authorgravatar for paul.verigo@gmail.comPavel Verigo <paul.verigo@gmail.com> 2026-03-06 08:27:24+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-03-11 22:46:58+01:00
log28022760919655354dd611455be0319862a3c409
treeadb70a391bff0cd68bc9e027c6178fc4884a2fd4
parent3069917384bd31dfb8e6c32e932222ab19609df1

stage2-wasm: address TODO in instruction selection code

This PR started as addressing the long-standing TODO above `buildOpcode`: /// TODO: deprecated, should be split up per tag. The code around this area was written a long time ago and has effectively become a legacy approach. When I started doing semi-occasional work for bringing >128-bit integer operations to the wasm backend, which is the last big missing piece of this backend, this design became annoying. While thinking about how to support that work, and also how vector unrolling should be handled (in cases where we do not rely purely on the legalize pass), I decided to do some architectural changes were needed. The first step is removing helpers like `intBinOp`, `floatBinOp`, `UnOp`, etc. They do not really capture all operations and resulted in a lot of small pieces of code trying to artificially unify different ops. Instead, the direction taken here is similar to `Sema/arith.zig`, introduce backend-oriented helpers such as `int*Op*Scalar` and `float*Op*` that operate purely in backend structures without referencing AIR at all. Additionally, the idea of introducing dedicated `IntType` and `FloatType` types was chosen. Using `Type` from Sema inside the backend is awkward, especially when strange or temporary types are needed. Creating them through `PerThread` is also undesirable since the backend strives to not modify `InternPool`. This goal is not fully achieved yet, some parts still require changes, and `InternPool` type formatting still requires `pt` for error reporting. This PR also enables legalize passes for some packed operations. The previous code in this area was buggy, and given the current state of the backend, relying on legalization is simpler. Finally, this PR disables one behavior test: `atomicrmw` with floats. The test seems to only run the non-concurrency path, because it would crash otherwise, and since we do not currently run behavior tests for the self-hosted backend with concurrency or atomics enabled, it does not provide meaningful coverage yet. In summary, this refactor reworks instruction selection in the wasm backend to simplify the code and make future work, especially adding big integer support.

4 files changed, 4184 insertions(+), 4588 deletions(-)

src/codegen/wasm/CodeGen.zig+4183-4582
...@@ -36,6 +36,11 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {...@@ -36,6 +36,11 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
36 .expand_add_safe,36 .expand_add_safe,
37 .expand_sub_safe,37 .expand_sub_safe,
38 .expand_mul_safe,38 .expand_mul_safe,
39
40 .expand_packed_load,
41 .expand_packed_store,
42 .expand_packed_struct_field_val,
43 .expand_packed_aggregate_init,
39 });44 });
40}45}
4146
...@@ -238,467 +243,8 @@ const WValue = union(enum) {...@@ -238,467 +243,8 @@ const WValue = union(enum) {
238 }243 }
239};244};
240245
241const Op = enum {
242 @"unreachable",
243 nop,
244 block,
245 loop,
246 @"if",
247 @"else",
248 end,
249 br,
250 br_if,
251 br_table,
252 @"return",
253 call,
254 drop,
255 select,
256 global_get,
257 global_set,
258 load,
259 store,
260 memory_size,
261 memory_grow,
262 @"const",
263 eqz,
264 eq,
265 ne,
266 lt,
267 gt,
268 le,
269 ge,
270 clz,
271 ctz,
272 popcnt,
273 add,
274 sub,
275 mul,
276 div,
277 rem,
278 @"and",
279 @"or",
280 xor,
281 shl,
282 shr,
283 rotl,
284 rotr,
285 abs,
286 neg,
287 ceil,
288 floor,
289 trunc,
290 nearest,
291 sqrt,
292 min,
293 max,
294 copysign,
295 wrap,
296 convert,
297 demote,
298 promote,
299 reinterpret,
300 extend,
301};
302
303const OpcodeBuildArguments = struct {
304 /// First valtype in the opcode (usually represents the type of the output)
305 valtype1: ?std.wasm.Valtype = null,
306 /// The operation (e.g. call, unreachable, div, min, sqrt, etc.)
307 op: Op,
308 /// Width of the operation (e.g. 8 for i32_load8_s, 16 for i64_extend16_i32_s)
309 width: ?u8 = null,
310 /// Second valtype in the opcode name (usually represents the type of the input)
311 valtype2: ?std.wasm.Valtype = null,
312 /// Signedness of the op
313 signedness: ?std.builtin.Signedness = null,
314};
315
316/// TODO: deprecated, should be split up per tag.
317fn buildOpcode(args: OpcodeBuildArguments) std.wasm.Opcode {
318 switch (args.op) {
319 .@"unreachable" => unreachable,
320 .nop => unreachable,
321 .block => unreachable,
322 .loop => unreachable,
323 .@"if" => unreachable,
324 .@"else" => unreachable,
325 .end => unreachable,
326 .br => unreachable,
327 .br_if => unreachable,
328 .br_table => unreachable,
329 .@"return" => unreachable,
330 .call => unreachable,
331 .drop => unreachable,
332 .select => unreachable,
333 .global_get => unreachable,
334 .global_set => unreachable,
335
336 .load => if (args.width) |width| switch (width) {
337 8 => switch (args.valtype1.?) {
338 .i32 => if (args.signedness.? == .signed) return .i32_load8_s else return .i32_load8_u,
339 .i64 => if (args.signedness.? == .signed) return .i64_load8_s else return .i64_load8_u,
340 .f32, .f64, .v128 => unreachable,
341 },
342 16 => switch (args.valtype1.?) {
343 .i32 => if (args.signedness.? == .signed) return .i32_load16_s else return .i32_load16_u,
344 .i64 => if (args.signedness.? == .signed) return .i64_load16_s else return .i64_load16_u,
345 .f32, .f64, .v128 => unreachable,
346 },
347 32 => switch (args.valtype1.?) {
348 .i64 => if (args.signedness.? == .signed) return .i64_load32_s else return .i64_load32_u,
349 .i32 => return .i32_load,
350 .f32 => return .f32_load,
351 .f64, .v128 => unreachable,
352 },
353 64 => switch (args.valtype1.?) {
354 .i64 => return .i64_load,
355 .f64 => return .f64_load,
356 else => unreachable,
357 },
358 else => unreachable,
359 } else switch (args.valtype1.?) {
360 .i32 => return .i32_load,
361 .i64 => return .i64_load,
362 .f32 => return .f32_load,
363 .f64 => return .f64_load,
364 .v128 => unreachable, // handled independently
365 },
366 .store => if (args.width) |width| {
367 switch (width) {
368 8 => switch (args.valtype1.?) {
369 .i32 => return .i32_store8,
370 .i64 => return .i64_store8,
371 .f32, .f64, .v128 => unreachable,
372 },
373 16 => switch (args.valtype1.?) {
374 .i32 => return .i32_store16,
375 .i64 => return .i64_store16,
376 .f32, .f64, .v128 => unreachable,
377 },
378 32 => switch (args.valtype1.?) {
379 .i64 => return .i64_store32,
380 .i32 => return .i32_store,
381 .f32 => return .f32_store,
382 .f64, .v128 => unreachable,
383 },
384 64 => switch (args.valtype1.?) {
385 .i64 => return .i64_store,
386 .f64 => return .f64_store,
387 else => unreachable,
388 },
389 else => unreachable,
390 }
391 } else {
392 switch (args.valtype1.?) {
393 .i32 => return .i32_store,
394 .i64 => return .i64_store,
395 .f32 => return .f32_store,
396 .f64 => return .f64_store,
397 .v128 => unreachable, // handled independently
398 }
399 },
400
401 .memory_size => return .memory_size,
402 .memory_grow => return .memory_grow,
403
404 .@"const" => switch (args.valtype1.?) {
405 .i32 => return .i32_const,
406 .i64 => return .i64_const,
407 .f32 => return .f32_const,
408 .f64 => return .f64_const,
409 .v128 => unreachable, // handled independently
410 },
411
412 .eqz => switch (args.valtype1.?) {
413 .i32 => return .i32_eqz,
414 .i64 => return .i64_eqz,
415 .f32, .f64, .v128 => unreachable,
416 },
417 .eq => switch (args.valtype1.?) {
418 .i32 => return .i32_eq,
419 .i64 => return .i64_eq,
420 .f32 => return .f32_eq,
421 .f64 => return .f64_eq,
422 .v128 => unreachable, // handled independently
423 },
424 .ne => switch (args.valtype1.?) {
425 .i32 => return .i32_ne,
426 .i64 => return .i64_ne,
427 .f32 => return .f32_ne,
428 .f64 => return .f64_ne,
429 .v128 => unreachable, // handled independently
430 },
431
432 .lt => switch (args.valtype1.?) {
433 .i32 => if (args.signedness.? == .signed) return .i32_lt_s else return .i32_lt_u,
434 .i64 => if (args.signedness.? == .signed) return .i64_lt_s else return .i64_lt_u,
435 .f32 => return .f32_lt,
436 .f64 => return .f64_lt,
437 .v128 => unreachable, // handled independently
438 },
439 .gt => switch (args.valtype1.?) {
440 .i32 => if (args.signedness.? == .signed) return .i32_gt_s else return .i32_gt_u,
441 .i64 => if (args.signedness.? == .signed) return .i64_gt_s else return .i64_gt_u,
442 .f32 => return .f32_gt,
443 .f64 => return .f64_gt,
444 .v128 => unreachable, // handled independently
445 },
446 .le => switch (args.valtype1.?) {
447 .i32 => if (args.signedness.? == .signed) return .i32_le_s else return .i32_le_u,
448 .i64 => if (args.signedness.? == .signed) return .i64_le_s else return .i64_le_u,
449 .f32 => return .f32_le,
450 .f64 => return .f64_le,
451 .v128 => unreachable, // handled independently
452 },
453 .ge => switch (args.valtype1.?) {
454 .i32 => if (args.signedness.? == .signed) return .i32_ge_s else return .i32_ge_u,
455 .i64 => if (args.signedness.? == .signed) return .i64_ge_s else return .i64_ge_u,
456 .f32 => return .f32_ge,
457 .f64 => return .f64_ge,
458 .v128 => unreachable, // handled independently
459 },
460
461 .clz => switch (args.valtype1.?) {
462 .i32 => return .i32_clz,
463 .i64 => return .i64_clz,
464 .f32, .f64 => unreachable,
465 .v128 => unreachable, // handled independently
466 },
467 .ctz => switch (args.valtype1.?) {
468 .i32 => return .i32_ctz,
469 .i64 => return .i64_ctz,
470 .f32, .f64 => unreachable,
471 .v128 => unreachable, // handled independently
472 },
473 .popcnt => switch (args.valtype1.?) {
474 .i32 => return .i32_popcnt,
475 .i64 => return .i64_popcnt,
476 .f32, .f64 => unreachable,
477 .v128 => unreachable, // handled independently
478 },
479
480 .add => switch (args.valtype1.?) {
481 .i32 => return .i32_add,
482 .i64 => return .i64_add,
483 .f32 => return .f32_add,
484 .f64 => return .f64_add,
485 .v128 => unreachable, // handled independently
486 },
487 .sub => switch (args.valtype1.?) {
488 .i32 => return .i32_sub,
489 .i64 => return .i64_sub,
490 .f32 => return .f32_sub,
491 .f64 => return .f64_sub,
492 .v128 => unreachable, // handled independently
493 },
494 .mul => switch (args.valtype1.?) {
495 .i32 => return .i32_mul,
496 .i64 => return .i64_mul,
497 .f32 => return .f32_mul,
498 .f64 => return .f64_mul,
499 .v128 => unreachable, // handled independently
500 },
501
502 .div => switch (args.valtype1.?) {
503 .i32 => if (args.signedness.? == .signed) return .i32_div_s else return .i32_div_u,
504 .i64 => if (args.signedness.? == .signed) return .i64_div_s else return .i64_div_u,
505 .f32 => return .f32_div,
506 .f64 => return .f64_div,
507 .v128 => unreachable, // handled independently
508 },
509 .rem => switch (args.valtype1.?) {
510 .i32 => if (args.signedness.? == .signed) return .i32_rem_s else return .i32_rem_u,
511 .i64 => if (args.signedness.? == .signed) return .i64_rem_s else return .i64_rem_u,
512 .f32, .f64 => unreachable,
513 .v128 => unreachable, // handled independently
514 },
515
516 .@"and" => switch (args.valtype1.?) {
517 .i32 => return .i32_and,
518 .i64 => return .i64_and,
519 .f32, .f64 => unreachable,
520 .v128 => unreachable, // handled independently
521 },
522 .@"or" => switch (args.valtype1.?) {
523 .i32 => return .i32_or,
524 .i64 => return .i64_or,
525 .f32, .f64 => unreachable,
526 .v128 => unreachable, // handled independently
527 },
528 .xor => switch (args.valtype1.?) {
529 .i32 => return .i32_xor,
530 .i64 => return .i64_xor,
531 .f32, .f64 => unreachable,
532 .v128 => unreachable, // handled independently
533 },
534
535 .shl => switch (args.valtype1.?) {
536 .i32 => return .i32_shl,
537 .i64 => return .i64_shl,
538 .f32, .f64 => unreachable,
539 .v128 => unreachable, // handled independently
540 },
541 .shr => switch (args.valtype1.?) {
542 .i32 => if (args.signedness.? == .signed) return .i32_shr_s else return .i32_shr_u,
543 .i64 => if (args.signedness.? == .signed) return .i64_shr_s else return .i64_shr_u,
544 .f32, .f64 => unreachable,
545 .v128 => unreachable, // handled independently
546 },
547 .rotl => switch (args.valtype1.?) {
548 .i32 => return .i32_rotl,
549 .i64 => return .i64_rotl,
550 .f32, .f64 => unreachable,
551 .v128 => unreachable, // handled independently
552 },
553 .rotr => switch (args.valtype1.?) {
554 .i32 => return .i32_rotr,
555 .i64 => return .i64_rotr,
556 .f32, .f64 => unreachable,
557 .v128 => unreachable, // handled independently
558 },
559
560 .abs => switch (args.valtype1.?) {
561 .i32, .i64 => unreachable,
562 .f32 => return .f32_abs,
563 .f64 => return .f64_abs,
564 .v128 => unreachable, // handled independently
565 },
566 .neg => switch (args.valtype1.?) {
567 .i32, .i64 => unreachable,
568 .f32 => return .f32_neg,
569 .f64 => return .f64_neg,
570 .v128 => unreachable, // handled independently
571 },
572 .ceil => switch (args.valtype1.?) {
573 .i64 => unreachable,
574 .i32 => return .f32_ceil, // when valtype is f16, we store it in i32.
575 .f32 => return .f32_ceil,
576 .f64 => return .f64_ceil,
577 .v128 => unreachable, // handled independently
578 },
579 .floor => switch (args.valtype1.?) {
580 .i64 => unreachable,
581 .i32 => return .f32_floor, // when valtype is f16, we store it in i32.
582 .f32 => return .f32_floor,
583 .f64 => return .f64_floor,
584 .v128 => unreachable, // handled independently
585 },
586 .trunc => switch (args.valtype1.?) {
587 .i32 => if (args.valtype2) |valty| switch (valty) {
588 .i32 => unreachable,
589 .i64 => unreachable,
590 .f32 => if (args.signedness.? == .signed) return .i32_trunc_f32_s else return .i32_trunc_f32_u,
591 .f64 => if (args.signedness.? == .signed) return .i32_trunc_f64_s else return .i32_trunc_f64_u,
592 .v128 => unreachable, // handled independently
593 } else return .f32_trunc, // when no valtype2, it's an f16 instead which is stored in an i32.
594 .i64 => switch (args.valtype2.?) {
595 .i32 => unreachable,
596 .i64 => unreachable,
597 .f32 => if (args.signedness.? == .signed) return .i64_trunc_f32_s else return .i64_trunc_f32_u,
598 .f64 => if (args.signedness.? == .signed) return .i64_trunc_f64_s else return .i64_trunc_f64_u,
599 .v128 => unreachable, // handled independently
600 },
601 .f32 => return .f32_trunc,
602 .f64 => return .f64_trunc,
603 .v128 => unreachable, // handled independently
604 },
605 .nearest => switch (args.valtype1.?) {
606 .i32, .i64 => unreachable,
607 .f32 => return .f32_nearest,
608 .f64 => return .f64_nearest,
609 .v128 => unreachable, // handled independently
610 },
611 .sqrt => switch (args.valtype1.?) {
612 .i32, .i64 => unreachable,
613 .f32 => return .f32_sqrt,
614 .f64 => return .f64_sqrt,
615 .v128 => unreachable, // handled independently
616 },
617 .min => switch (args.valtype1.?) {
618 .i32, .i64 => unreachable,
619 .f32 => return .f32_min,
620 .f64 => return .f64_min,
621 .v128 => unreachable, // handled independently
622 },
623 .max => switch (args.valtype1.?) {
624 .i32, .i64 => unreachable,
625 .f32 => return .f32_max,
626 .f64 => return .f64_max,
627 .v128 => unreachable, // handled independently
628 },
629 .copysign => switch (args.valtype1.?) {
630 .i32, .i64 => unreachable,
631 .f32 => return .f32_copysign,
632 .f64 => return .f64_copysign,
633 .v128 => unreachable, // handled independently
634 },
635
636 .wrap => switch (args.valtype1.?) {
637 .i32 => switch (args.valtype2.?) {
638 .i32 => unreachable,
639 .i64 => return .i32_wrap_i64,
640 .f32, .f64 => unreachable,
641 .v128 => unreachable, // handled independently
642 },
643 .i64, .f32, .f64 => unreachable,
644 .v128 => unreachable, // handled independently
645 },
646 .convert => switch (args.valtype1.?) {
647 .i32, .i64 => unreachable,
648 .f32 => switch (args.valtype2.?) {
649 .i32 => if (args.signedness.? == .signed) return .f32_convert_i32_s else return .f32_convert_i32_u,
650 .i64 => if (args.signedness.? == .signed) return .f32_convert_i64_s else return .f32_convert_i64_u,
651 .f32, .f64 => unreachable,
652 .v128 => unreachable, // handled independently
653 },
654 .f64 => switch (args.valtype2.?) {
655 .i32 => if (args.signedness.? == .signed) return .f64_convert_i32_s else return .f64_convert_i32_u,
656 .i64 => if (args.signedness.? == .signed) return .f64_convert_i64_s else return .f64_convert_i64_u,
657 .f32, .f64 => unreachable,
658 .v128 => unreachable, // handled independently
659 },
660 .v128 => unreachable, // handled independently
661 },
662 .demote => if (args.valtype1.? == .f32 and args.valtype2.? == .f64) return .f32_demote_f64 else unreachable,
663 .promote => if (args.valtype1.? == .f64 and args.valtype2.? == .f32) return .f64_promote_f32 else unreachable,
664 .reinterpret => switch (args.valtype1.?) {
665 .i32 => if (args.valtype2.? == .f32) return .i32_reinterpret_f32 else unreachable,
666 .i64 => if (args.valtype2.? == .f64) return .i64_reinterpret_f64 else unreachable,
667 .f32 => if (args.valtype2.? == .i32) return .f32_reinterpret_i32 else unreachable,
668 .f64 => if (args.valtype2.? == .i64) return .f64_reinterpret_i64 else unreachable,
669 .v128 => unreachable, // handled independently
670 },
671 .extend => switch (args.valtype1.?) {
672 .i32 => switch (args.width.?) {
673 8 => if (args.signedness.? == .signed) return .i32_extend8_s else unreachable,
674 16 => if (args.signedness.? == .signed) return .i32_extend16_s else unreachable,
675 else => unreachable,
676 },
677 .i64 => switch (args.width.?) {
678 8 => if (args.signedness.? == .signed) return .i64_extend8_s else unreachable,
679 16 => if (args.signedness.? == .signed) return .i64_extend16_s else unreachable,
680 32 => if (args.signedness.? == .signed) return .i64_extend32_s else unreachable,
681 else => unreachable,
682 },
683 .f32, .f64 => unreachable,
684 .v128 => unreachable, // handled independently
685 },
686 }
687}
688
689test "Wasm - buildOpcode" {
690 // Make sure buildOpcode is referenced, and test some examples
691 const i32_const = buildOpcode(.{ .op = .@"const", .valtype1 = .i32 });
692 const i64_extend32_s = buildOpcode(.{ .op = .extend, .valtype1 = .i64, .width = 32, .signedness = .signed });
693 const f64_reinterpret_i64 = buildOpcode(.{ .op = .reinterpret, .valtype1 = .f64, .valtype2 = .i64 });
694
695 try testing.expectEqual(@as(std.wasm.Opcode, .i32_const), i32_const);
696 try testing.expectEqual(@as(std.wasm.Opcode, .i64_extend32_s), i64_extend32_s);
697 try testing.expectEqual(@as(std.wasm.Opcode, .f64_reinterpret_i64), f64_reinterpret_i64);
698}
699
700/// Hashmap to store generated `WValue` for each `Air.Inst.Ref`246/// Hashmap to store generated `WValue` for each `Air.Inst.Ref`
701pub const ValueTable = std.AutoArrayHashMapUnmanaged(Air.Inst.Ref, WValue);247const ValueTable = std.AutoArrayHashMapUnmanaged(Air.Inst.Ref, WValue);
702248
703const bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};249const bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
704250
...@@ -1496,13 +1042,6 @@ fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue {...@@ -1496,13 +1042,6 @@ fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue {
1496 return .{ .stack_offset = .{ .value = offset, .references = 1 } };1042 return .{ .stack_offset = .{ .value = offset, .references = 1 } };
1497}1043}
14981044
1499/// From given zig bitsize, returns the wasm bitsize
1500fn toWasmBits(bits: u16) ?u16 {
1501 return for ([_]u16{ 32, 64, 128 }) |wasm_bits| {
1502 if (bits <= wasm_bits) return wasm_bits;
1503 } else null;
1504}
1505
1506/// Performs a copy of bytes for a given type. Copying all bytes1045/// Performs a copy of bytes for a given type. Copying all bytes
1507/// from rhs to lhs.1046/// from rhs to lhs.
1508fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {1047fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
...@@ -1760,6 +1299,7 @@ fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum...@@ -1760,6 +1299,7 @@ fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum
1760}1299}
17611300
1762fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {1301fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1302 const zcu = cg.pt.zcu;
1763 const air_tags = cg.air.instructions.items(.tag);1303 const air_tags = cg.air.instructions.items(.tag);
1764 return switch (air_tags[@intFromEnum(inst)]) {1304 return switch (air_tags[@intFromEnum(inst)]) {
1765 // No "scalarize" legalizations are enabled, so these instructions never appear.1305 // No "scalarize" legalizations are enabled, so these instructions never appear.
...@@ -1770,57 +1310,405 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1770,57 +1310,405 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
17701310
1771 .inferred_alloc, .inferred_alloc_comptime => unreachable,1311 .inferred_alloc, .inferred_alloc_comptime => unreachable,
17721312
1773 .add => cg.airBinOp(inst, .add),1313 .add,
1774 .add_sat => cg.airSatBinOp(inst, .add),1314 .sub,
1775 .add_wrap => cg.airWrapBinOp(inst, .add),1315 .mul,
1776 .sub => cg.airBinOp(inst, .sub),1316 .rem,
1777 .sub_sat => cg.airSatBinOp(inst, .sub),1317 .mod,
1778 .sub_wrap => cg.airWrapBinOp(inst, .sub),1318 .max,
1779 .mul => cg.airBinOp(inst, .mul),1319 .min,
1780 .mul_sat => cg.airSatMul(inst),1320 .div_trunc,
1781 .mul_wrap => cg.airWrapBinOp(inst, .mul),1321 .div_floor,
1782 .div_float, .div_exact => cg.airDiv(inst),1322 => |tag| {
1783 .div_trunc => cg.airDivTrunc(inst),1323 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1784 .div_floor => cg.airDivFloor(inst),1324 const lhs = try cg.resolveInst(bin_op.lhs);
1785 .bit_and => cg.airBinOp(inst, .@"and"),1325 const rhs = try cg.resolveInst(bin_op.rhs);
1786 .bit_or => cg.airBinOp(inst, .@"or"),1326
1787 .bool_and => cg.airBinOp(inst, .@"and"),1327 const ty = cg.typeOfIndex(inst);
1788 .bool_or => cg.airBinOp(inst, .@"or"),1328 const type_tag = ty.zigTypeTag(zcu);
1789 .rem => cg.airRem(inst),1329
1790 .mod => cg.airMod(inst),1330 if (type_tag == .vector) {
1791 .shl => cg.airWrapBinOp(inst, .shl),1331 return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1792 .shl_exact => cg.airBinOp(inst, .shl),1332 }
1793 .shl_sat => cg.airShlSat(inst),1333
1794 .shr, .shr_exact => cg.airBinOp(inst, .shr),1334 if (type_tag == .float) {
1795 .xor => cg.airBinOp(inst, .xor),1335 const float_ty: FloatType = .fromType(cg, ty);
1796 .max => cg.airMaxMin(inst, .fmax, .gt),1336
1797 .min => cg.airMaxMin(inst, .fmin, .lt),1337 const result = switch (tag) {
1798 .mul_add => cg.airMulAdd(inst),1338 .add => try cg.floatAdd(float_ty, lhs, rhs),
17991339 .sub => try cg.floatSub(float_ty, lhs, rhs),
1800 .sqrt => cg.airUnaryFloatOp(inst, .sqrt),1340 .mul => try cg.floatMul(float_ty, lhs, rhs),
1801 .sin => cg.airUnaryFloatOp(inst, .sin),1341 .rem => try cg.floatRem(float_ty, lhs, rhs),
1802 .cos => cg.airUnaryFloatOp(inst, .cos),1342 .mod => try cg.floatMod(float_ty, lhs, rhs),
1803 .tan => cg.airUnaryFloatOp(inst, .tan),1343 .max => try cg.floatMax(float_ty, lhs, rhs),
1804 .exp => cg.airUnaryFloatOp(inst, .exp),1344 .min => try cg.floatMin(float_ty, lhs, rhs),
1805 .exp2 => cg.airUnaryFloatOp(inst, .exp2),1345 .div_trunc => try cg.floatDivTrunc(float_ty, lhs, rhs),
1806 .log => cg.airUnaryFloatOp(inst, .log),1346 .div_floor => try cg.floatDivFloor(float_ty, lhs, rhs),
1807 .log2 => cg.airUnaryFloatOp(inst, .log2),1347 else => unreachable,
1808 .log10 => cg.airUnaryFloatOp(inst, .log10),1348 };
1809 .floor => cg.airUnaryFloatOp(inst, .floor),1349
1810 .ceil => cg.airUnaryFloatOp(inst, .ceil),1350 try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1811 .round => cg.airUnaryFloatOp(inst, .round),1351 } else if (type_tag == .int) {
1812 .trunc_float => cg.airUnaryFloatOp(inst, .trunc),1352 const int_ty: IntType = .fromType(cg, ty);
1813 .neg => cg.airUnaryFloatOp(inst, .neg),1353
18141354 const result = switch (tag) {
1815 .abs => cg.airAbs(inst),1355 .add => try cg.intAdd(int_ty, lhs, rhs),
18161356 .sub => try cg.intSub(int_ty, lhs, rhs),
1817 .add_with_overflow => cg.airAddSubWithOverflow(inst, .add),1357 .mul => try cg.intMul(int_ty, lhs, rhs),
1818 .sub_with_overflow => cg.airAddSubWithOverflow(inst, .sub),1358 .rem => try cg.intRem(int_ty, lhs, rhs),
1819 .shl_with_overflow => cg.airShlWithOverflow(inst),1359 .mod => try cg.intMod(int_ty, lhs, rhs),
1820 .mul_with_overflow => cg.airMulWithOverflow(inst),1360 .max => try cg.intMax(int_ty, lhs, rhs),
18211361 .min => try cg.intMin(int_ty, lhs, rhs),
1822 .clz => cg.airClz(inst),1362 .div_trunc => try cg.intDiv(int_ty, lhs, rhs),
1823 .ctz => cg.airCtz(inst),1363 .div_floor => try cg.intDivFloor(int_ty, lhs, rhs),
1364 else => unreachable,
1365 };
1366
1367 try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1368 } else {
1369 unreachable;
1370 }
1371 },
1372 .div_float => {
1373 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1374 const lhs = try cg.resolveInst(bin_op.lhs);
1375 const rhs = try cg.resolveInst(bin_op.rhs);
1376 const ty = cg.typeOfIndex(inst);
1377
1378 if (ty.zigTypeTag(zcu) == .vector) {
1379 return cg.fail("TODO: implement AIR op: div_float for vectors", .{});
1380 }
1381
1382 const result = try cg.floatDiv(.fromType(cg, ty), lhs, rhs);
1383 try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1384 },
1385 .div_exact => {
1386 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1387 const lhs = try cg.resolveInst(bin_op.lhs);
1388 const rhs = try cg.resolveInst(bin_op.rhs);
1389 const ty = cg.typeOfIndex(inst);
1390
1391 if (ty.zigTypeTag(zcu) == .vector) {
1392 return cg.fail("TODO: implement AIR op: div_exact for vectors", .{});
1393 }
1394
1395 const result = try cg.intDiv(.fromType(cg, ty), lhs, rhs);
1396 try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1397 },
1398 .abs => {
1399 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1400 const operand = try cg.resolveInst(ty_op.operand);
1401
1402 const ty = cg.typeOf(ty_op.operand);
1403 const type_tag = ty.zigTypeTag(zcu);
1404
1405 if (type_tag == .vector) {
1406 return cg.fail("TODO: implement AIR op: abs for vectors", .{});
1407 }
1408
1409 if (type_tag == .float) {
1410 const result = try cg.floatAbs(.fromType(cg, ty), operand);
1411 return cg.finishAir(inst, result, &.{ty_op.operand});
1412 } else if (type_tag == .int) {
1413 const result = try cg.intAbs(.fromType(cg, ty), operand);
1414 return cg.finishAir(inst, result, &.{ty_op.operand});
1415 } else {
1416 unreachable;
1417 }
1418 },
1419 .mul_add => {
1420 const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
1421 const bin_op = cg.air.extraData(Air.Bin, pl_op.payload).data;
1422 const addend = try cg.resolveInst(pl_op.operand);
1423 const lhs = try cg.resolveInst(bin_op.lhs);
1424 const rhs = try cg.resolveInst(bin_op.rhs);
1425 const ty = cg.typeOfIndex(inst);
1426
1427 if (ty.zigTypeTag(cg.pt.zcu) == .vector) {
1428 return cg.fail("TODO: implement AIR op: mul_add for vectors", .{});
1429 }
1430
1431 const result = try cg.floatMulAdd(.fromType(cg, ty), lhs, rhs, addend);
1432 return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs, pl_op.operand });
1433 },
1434
1435 .add_sat,
1436 .sub_sat,
1437 .mul_sat,
1438 .shl_sat,
1439 => |tag| {
1440 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1441 const lhs = try cg.resolveInst(bin_op.lhs);
1442 const rhs = try cg.resolveInst(bin_op.rhs);
1443 const ty = cg.typeOfIndex(inst);
1444
1445 if (ty.zigTypeTag(cg.pt.zcu) == .vector) {
1446 return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1447 }
1448
1449 const int_ty: IntType = .fromType(cg, ty);
1450 const result = switch (tag) {
1451 .add_sat => try cg.intAddSat(int_ty, lhs, rhs),
1452 .sub_sat => try cg.intSubSat(int_ty, lhs, rhs),
1453 .mul_sat => try cg.intMulSat(int_ty, lhs, rhs),
1454 .shl_sat => try cg.intShlSat(int_ty, lhs, rhs),
1455 else => unreachable,
1456 };
1457
1458 try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1459 },
1460
1461 .add_with_overflow,
1462 .sub_with_overflow,
1463 .mul_with_overflow,
1464 .shl_with_overflow,
1465 => |tag| {
1466 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1467 const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data;
1468
1469 const lhs = try cg.resolveInst(extra.lhs);
1470 const rhs = try cg.resolveInst(extra.rhs);
1471
1472 const ty = cg.typeOf(extra.lhs);
1473 const int_ty: IntType = .fromType(cg, ty);
1474
1475 const out = switch (tag) {
1476 .add_with_overflow => try cg.intAddOverflow(int_ty, lhs, rhs),
1477 .sub_with_overflow => try cg.intSubOverflow(int_ty, lhs, rhs),
1478 .mul_with_overflow => try cg.intMulOverflow(int_ty, lhs, rhs),
1479 .shl_with_overflow => try cg.intShlOverflow(int_ty, lhs, rhs),
1480 else => unreachable,
1481 };
1482
1483 var ov_tmp = try out.ov.toLocal(cg, Type.u1);
1484 defer ov_tmp.free(cg);
1485
1486 var res_tmp = try out.result.toLocal(cg, ty);
1487 defer res_tmp.free(cg);
1488
1489 const result = try cg.allocStack(cg.typeOfIndex(inst));
1490 const offset: u32 = @intCast(ty.abiSize(cg.pt.zcu));
1491
1492 try cg.store(result, res_tmp, ty, 0);
1493 try cg.store(result, ov_tmp, Type.u1, offset);
1494
1495 try cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs });
1496 },
1497
1498 .add_wrap, .sub_wrap, .mul_wrap, .shl => |tag| {
1499 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1500 const lhs = try cg.resolveInst(bin_op.lhs);
1501 const rhs = try cg.resolveInst(bin_op.rhs);
1502 const ty = cg.typeOfIndex(inst);
1503
1504 if (ty.zigTypeTag(zcu) == .vector) {
1505 return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1506 }
1507
1508 const int_ty: IntType = .fromType(cg, ty);
1509 const raw_result = switch (tag) {
1510 .add_wrap => try cg.intAdd(int_ty, lhs, rhs),
1511 .sub_wrap => try cg.intSub(int_ty, lhs, rhs),
1512 .mul_wrap => try cg.intMul(int_ty, lhs, rhs),
1513 .shl => try cg.intShl(int_ty, lhs, rhs),
1514 else => unreachable,
1515 };
1516 const result = try cg.intWrap(int_ty, raw_result);
1517
1518 try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1519 },
1520
1521 .bit_and, .bit_or, .bool_and, .bool_or, .xor, .shl_exact, .shr, .shr_exact => |tag| {
1522 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1523 const lhs = try cg.resolveInst(bin_op.lhs);
1524 const rhs = try cg.resolveInst(bin_op.rhs);
1525 const ty = cg.typeOfIndex(inst);
1526
1527 if (ty.zigTypeTag(zcu) == .vector) {
1528 return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1529 }
1530
1531 const int_ty: IntType = .fromType(cg, ty);
1532 const result = switch (tag) {
1533 .bit_and, .bool_and => try cg.intAnd(int_ty, lhs, rhs),
1534 .bit_or, .bool_or => try cg.intOr(int_ty, lhs, rhs),
1535 .xor => try cg.intXor(int_ty, lhs, rhs),
1536 .shl_exact => try cg.intShl(int_ty, lhs, rhs),
1537 .shr, .shr_exact => try cg.intShr(int_ty, lhs, rhs),
1538 else => unreachable,
1539 };
1540
1541 try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1542 },
1543
1544 .not => {
1545 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1546 const operand = try cg.resolveInst(ty_op.operand);
1547 const ty = cg.typeOf(ty_op.operand);
1548
1549 if (ty.zigTypeTag(zcu) == .vector) {
1550 return cg.fail("TODO: implement AIR op: not for vectors", .{});
1551 }
1552
1553 const result = try cg.intNot(.fromType(cg, ty), operand);
1554 try cg.finishAir(inst, result, &.{ty_op.operand});
1555 },
1556
1557 .bitcast => cg.airBitcast(inst),
1558
1559 .intcast => {
1560 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1561
1562 const dest_ty = ty_op.ty.toType();
1563 const operand = try cg.resolveInst(ty_op.operand);
1564 const src_ty = cg.typeOf(ty_op.operand);
1565
1566 if (dest_ty.zigTypeTag(zcu) == .vector) {
1567 return cg.fail("TODO: implement AIR op: intcast for vectors", .{});
1568 }
1569
1570 const src_int_ty: IntType = .fromType(cg, src_ty);
1571 const dest_int_ty: IntType = .fromType(cg, dest_ty);
1572
1573 const src_bits = src_int_ty.bits;
1574 const dest_bits = dest_int_ty.bits;
1575
1576 const same_class: bool = (src_bits <= 32 and dest_bits <= 32) or
1577 (src_bits >= 33 and src_bits <= 64 and dest_bits >= 33 and dest_bits <= 64) or
1578 (src_bits >= 65 and src_bits <= 128 and dest_bits >= 65 and dest_bits <= 128);
1579
1580 const result = if (same_class)
1581 cg.reuseOperand(ty_op.operand, operand)
1582 else
1583 try cg.intCast(dest_int_ty, src_int_ty, operand);
1584
1585 try cg.finishAir(inst, result, &.{ty_op.operand});
1586 },
1587 .trunc => {
1588 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1589
1590 const operand = try cg.resolveInst(ty_op.operand);
1591 const dest_ty = ty_op.ty.toType();
1592 const src_ty = cg.typeOf(ty_op.operand);
1593
1594 if (dest_ty.zigTypeTag(zcu) == .vector or src_ty.zigTypeTag(zcu) == .vector) {
1595 return cg.fail("TODO: implement AIR op: trunc for vectors", .{});
1596 }
1597
1598 const src_int_ty: IntType = .fromType(cg, src_ty);
1599 const dest_int_ty: IntType = .fromType(cg, dest_ty);
1600
1601 const result = if (src_int_ty.bits == dest_int_ty.bits)
1602 cg.reuseOperand(ty_op.operand, operand)
1603 else blk: {
1604 break :blk try cg.intTrunc(dest_int_ty, src_int_ty, operand);
1605 };
1606
1607 try cg.finishAir(inst, result, &.{ty_op.operand});
1608 },
1609
1610 .fptrunc, .fpext => |tag| {
1611 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1612
1613 const operand = try cg.resolveInst(ty_op.operand);
1614 const src_ty = cg.typeOf(ty_op.operand);
1615 const dest_ty = cg.typeOfIndex(inst);
1616
1617 if (dest_ty.zigTypeTag(cg.pt.zcu) == .vector) {
1618 return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1619 }
1620
1621 const src_float_ty: FloatType = .fromType(cg, src_ty);
1622 const dest_float_ty: FloatType = .fromType(cg, dest_ty);
1623
1624 const result = switch (tag) {
1625 .fptrunc => try cg.floatTruncCast(dest_float_ty, src_float_ty, operand),
1626 .fpext => try cg.floatExtendCast(dest_float_ty, src_float_ty, operand),
1627 else => unreachable,
1628 };
1629
1630 try cg.finishAir(inst, result, &.{ty_op.operand});
1631 },
1632
1633 .int_from_float => {
1634 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1635 const operand = try cg.resolveInst(ty_op.operand);
1636 const src_ty = cg.typeOf(ty_op.operand);
1637 const dest_ty = cg.typeOfIndex(inst);
1638
1639 if (src_ty.zigTypeTag(zcu) == .vector) {
1640 return cg.fail("TODO: implement AIR op: int_from_float for vectors", .{});
1641 }
1642
1643 const result = try cg.intFromFloat(.fromType(cg, dest_ty), .fromType(cg, src_ty), operand);
1644 try cg.finishAir(inst, result, &.{ty_op.operand});
1645 },
1646 .float_from_int => {
1647 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1648 const operand = try cg.resolveInst(ty_op.operand);
1649 const src_ty = cg.typeOf(ty_op.operand);
1650 const dest_ty = cg.typeOfIndex(inst);
1651
1652 if (src_ty.zigTypeTag(zcu) == .vector) {
1653 return cg.fail("TODO: implement AIR op: float_from_int for vectors", .{});
1654 }
1655
1656 const result = try cg.floatFromInt(.fromType(cg, dest_ty), .fromType(cg, src_ty), operand);
1657 try cg.finishAir(inst, result, &.{ty_op.operand});
1658 },
1659
1660 .clz, .ctz, .popcount, .byte_swap, .bit_reverse => |tag| {
1661 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1662 const operand = try cg.resolveInst(ty_op.operand);
1663
1664 const ty = cg.typeOf(ty_op.operand);
1665
1666 if (ty.zigTypeTag(zcu) == .vector) {
1667 return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1668 }
1669
1670 const int_ty: IntType = .fromType(cg, ty);
1671 const result = switch (tag) {
1672 .clz => try cg.intClz(int_ty, operand),
1673 .ctz => try cg.intCtz(int_ty, operand),
1674 .popcount => try cg.intPopCount(int_ty, operand),
1675 .byte_swap => try cg.intByteSwap(int_ty, operand),
1676 .bit_reverse => try cg.intBitReverse(int_ty, operand),
1677 else => unreachable,
1678 };
1679 try cg.finishAir(inst, result, &.{ty_op.operand});
1680 },
1681
1682 .sqrt, .sin, .cos, .tan, .exp, .exp2, .log, .log2, .log10, .floor, .ceil, .round, .trunc_float, .neg => |tag| {
1683 const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1684 const operand = try cg.resolveInst(un_op);
1685 const ty = cg.typeOfIndex(inst);
1686
1687 if (ty.zigTypeTag(zcu) == .vector) {
1688 return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1689 }
1690
1691 const float_ty: FloatType = .fromType(cg, ty);
1692 const result = switch (tag) {
1693 .sqrt => try cg.floatSqrt(float_ty, operand),
1694 .sin => try cg.floatSin(float_ty, operand),
1695 .cos => try cg.floatCos(float_ty, operand),
1696 .tan => try cg.floatTan(float_ty, operand),
1697 .exp => try cg.floatExp(float_ty, operand),
1698 .exp2 => try cg.floatExp2(float_ty, operand),
1699 .log => try cg.floatLog(float_ty, operand),
1700 .log2 => try cg.floatLog2(float_ty, operand),
1701 .log10 => try cg.floatLog10(float_ty, operand),
1702 .floor => try cg.floatFloor(float_ty, operand),
1703 .ceil => try cg.floatCeil(float_ty, operand),
1704 .round => try cg.floatRound(float_ty, operand),
1705 .trunc_float => try cg.floatTrunc(float_ty, operand),
1706 .neg => try cg.floatNeg(float_ty, operand),
1707 else => unreachable,
1708 };
1709
1710 try cg.finishAir(inst, result, &.{un_op});
1711 },
18241712
1825 .cmp_eq => cg.airCmp(inst, .eq),1713 .cmp_eq => cg.airCmp(inst, .eq),
1826 .cmp_gte => cg.airCmp(inst, .gte),1714 .cmp_gte => cg.airCmp(inst, .gte),
...@@ -1836,20 +1724,14 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1836,20 +1724,14 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1836 .array_to_slice => cg.airArrayToSlice(inst),1724 .array_to_slice => cg.airArrayToSlice(inst),
1837 .alloc => cg.airAlloc(inst),1725 .alloc => cg.airAlloc(inst),
1838 .arg => cg.airArg(inst),1726 .arg => cg.airArg(inst),
1839 .bitcast => cg.airBitcast(inst),
1840 .block => cg.airBlock(inst),1727 .block => cg.airBlock(inst),
1841 .trap => cg.airTrap(inst),1728 .trap => cg.airTrap(inst),
1729 .unreach => cg.airUnreachable(inst),
1842 .breakpoint => cg.airBreakpoint(inst),1730 .breakpoint => cg.airBreakpoint(inst),
1843 .br => cg.airBr(inst),1731 .br => cg.airBr(inst),
1844 .repeat => cg.airRepeat(inst),1732 .repeat => cg.airRepeat(inst),
1845 .switch_dispatch => cg.airSwitchDispatch(inst),1733 .switch_dispatch => cg.airSwitchDispatch(inst),
1846 .cond_br => cg.airCondBr(inst),1734 .cond_br => cg.airCondBr(inst),
1847 .intcast => cg.airIntcast(inst),
1848 .fptrunc => cg.airFptrunc(inst),
1849 .fpext => cg.airFpext(inst),
1850 .int_from_float => cg.airIntFromFloat(inst),
1851 .float_from_int => cg.airFloatFromInt(inst),
1852 .get_union_tag => cg.airGetUnionTag(inst),
18531735
1854 .@"try" => cg.airTry(inst),1736 .@"try" => cg.airTry(inst),
1855 .try_cold => cg.airTry(inst),1737 .try_cold => cg.airTry(inst),
...@@ -1882,7 +1764,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1882,7 +1764,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1882 .loop => cg.airLoop(inst),1764 .loop => cg.airLoop(inst),
1883 .memset => cg.airMemset(inst, false),1765 .memset => cg.airMemset(inst, false),
1884 .memset_safe => cg.airMemset(inst, true),1766 .memset_safe => cg.airMemset(inst, true),
1885 .not => cg.airNot(inst),
1886 .optional_payload => cg.airOptionalPayload(inst),1767 .optional_payload => cg.airOptionalPayload(inst),
1887 .optional_payload_ptr => cg.airOptionalPayloadPtr(inst),1768 .optional_payload_ptr => cg.airOptionalPayloadPtr(inst),
1888 .optional_payload_ptr_set => cg.airOptionalPayloadPtrSet(inst),1769 .optional_payload_ptr_set => cg.airOptionalPayloadPtrSet(inst),
...@@ -1902,9 +1783,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1902,9 +1783,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1902 .aggregate_init => cg.airAggregateInit(inst),1783 .aggregate_init => cg.airAggregateInit(inst),
1903 .union_init => cg.airUnionInit(inst),1784 .union_init => cg.airUnionInit(inst),
1904 .prefetch => cg.airPrefetch(inst),1785 .prefetch => cg.airPrefetch(inst),
1905 .popcount => cg.airPopcount(inst),
1906 .byte_swap => cg.airByteSwap(inst),
1907 .bit_reverse => cg.airBitReverse(inst),
19081786
1909 .slice => cg.airSlice(inst),1787 .slice => cg.airSlice(inst),
1910 .slice_len => cg.airSliceLen(inst),1788 .slice_len => cg.airSliceLen(inst),
...@@ -1917,6 +1795,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1917,6 +1795,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1917 .store_safe => cg.airStore(inst, true),1795 .store_safe => cg.airStore(inst, true),
19181796
1919 .set_union_tag => cg.airSetUnionTag(inst),1797 .set_union_tag => cg.airSetUnionTag(inst),
1798 .get_union_tag => cg.airGetUnionTag(inst),
1920 .struct_field_ptr => cg.airStructFieldPtr(inst),1799 .struct_field_ptr => cg.airStructFieldPtr(inst),
1921 .struct_field_ptr_index_0 => cg.airStructFieldPtrIndex(inst, 0),1800 .struct_field_ptr_index_0 => cg.airStructFieldPtrIndex(inst, 0),
1922 .struct_field_ptr_index_1 => cg.airStructFieldPtrIndex(inst, 1),1801 .struct_field_ptr_index_1 => cg.airStructFieldPtrIndex(inst, 1),
...@@ -1927,8 +1806,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1927,8 +1806,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
19271806
1928 .switch_br => cg.airSwitchBr(inst, false),1807 .switch_br => cg.airSwitchBr(inst, false),
1929 .loop_switch_br => cg.airSwitchBr(inst, true),1808 .loop_switch_br => cg.airSwitchBr(inst, true),
1930 .trunc => cg.airTrunc(inst),
1931 .unreach => cg.airUnreachable(inst),
19321809
1933 .wrap_optional => cg.airWrapOptional(inst),1810 .wrap_optional => cg.airWrapOptional(inst),
1934 .unwrap_errunion_payload => cg.airUnwrapErrUnionPayload(inst, false),1811 .unwrap_errunion_payload => cg.airUnwrapErrUnionPayload(inst, false),
...@@ -1954,7 +1831,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1954,7 +1831,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1954 .runtime_nav_ptr => cg.airRuntimeNavPtr(inst),1831 .runtime_nav_ptr => cg.airRuntimeNavPtr(inst),
19551832
1956 .assembly,1833 .assembly,
1957
1958 .err_return_trace,1834 .err_return_trace,
1959 .set_err_return_trace,1835 .set_err_return_trace,
1960 .save_err_return_trace_index,1836 .save_err_return_trace_index,
...@@ -2230,62 +2106,9 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {...@@ -2230,62 +2106,9 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
2230 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });2106 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2231 }2107 }
22322108
2233 if (ptr_info.packed_offset.host_size == 0) {2109 assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store
2234 try cg.store(lhs, rhs, ty, 0);
2235 } else {
2236 // at this point we have a non-natural alignment, we must
2237 // load the value, and then shift+or the rhs into the result location.
2238 const host_size = ptr_info.packed_offset.host_size * 8;
2239 const host_ty = try pt.intType(.unsigned, host_size);
2240 const bit_size: u16 = @intCast(ty.bitSize(zcu));
2241 const bit_offset = ptr_info.packed_offset.bit_offset;
2242
2243 const mask_val = try cg.resolveValue(val: {
2244 const limbs = try cg.gpa.alloc(
2245 std.math.big.Limb,
2246 std.math.big.int.calcTwosCompLimbCount(host_size) + 1,
2247 );
2248 defer cg.gpa.free(limbs);
2249
2250 var mask_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
2251 mask_bigint.setTwosCompIntLimit(.max, .unsigned, host_size);
22522110
2253 if (bit_size != host_size) {2111 try cg.store(lhs, rhs, ty, 0);
2254 mask_bigint.shiftRight(mask_bigint.toConst(), host_size - bit_size);
2255 }
2256 if (bit_offset != 0) {
2257 mask_bigint.shiftLeft(mask_bigint.toConst(), bit_offset);
2258 }
2259 mask_bigint.bitNotWrap(mask_bigint.toConst(), .unsigned, host_size);
2260
2261 break :val try pt.intValue_big(host_ty, mask_bigint.toConst());
2262 });
2263
2264 const shift_val: WValue = if (33 <= host_size and host_size <= 64)
2265 .{ .imm64 = bit_offset }
2266 else
2267 .{ .imm32 = bit_offset };
2268
2269 if (host_size <= 64) {
2270 try cg.emitWValue(lhs);
2271 }
2272 const loaded = if (host_size <= 64)
2273 try cg.load(lhs, host_ty, 0)
2274 else
2275 lhs;
2276 const anded = try cg.binOp(loaded, mask_val, host_ty, .@"and");
2277 const extended_value = try cg.intcast(rhs, ty, host_ty);
2278 const shifted_value = if (bit_offset > 0)
2279 try cg.binOp(extended_value, shift_val, host_ty, .shl)
2280 else
2281 extended_value;
2282 const result = try cg.binOp(anded, shifted_value, host_ty, .@"or");
2283 if (host_size <= 64) {
2284 try cg.store(.stack, result, host_ty, lhs.offset());
2285 } else {
2286 try cg.store(lhs, result, host_ty, lhs.offset());
2287 }
2288 }
22892112
2290 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });2113 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2291}2114}
...@@ -2298,106 +2121,48 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr...@@ -2298,106 +2121,48 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
22982121
2299 if (!ty.hasRuntimeBits(zcu)) return;2122 if (!ty.hasRuntimeBits(zcu)) return;
23002123
2301 switch (ty.zigTypeTag(zcu)) {2124 if (isByRef(ty, zcu, cg.target)) {
2302 .error_union => {2125 return cg.memcpy(lhs, rhs, .{ .imm32 = @intCast(abi_size) });
2303 const pl_ty = ty.errorUnionPayload(zcu);2126 }
2304 if (!pl_ty.hasRuntimeBits(zcu)) {
2305 return cg.store(lhs, rhs, Type.anyerror, offset);
2306 }
23072127
2308 const len = @as(u32, @intCast(abi_size));2128 if (ty.zigTypeTag(zcu) == .vector) {
2309 assert(offset == 0);2129 try cg.emitWValue(lhs);
2310 return cg.memcpy(lhs, rhs, .{ .imm32 = len });2130 try cg.lowerToStack(rhs);
2311 },2131 // TODO: Add helper functions for simd opcodes
2312 .optional => {2132 const extra_index: u32 = @intCast(cg.mir_extra.items.len);
2313 if (ty.isPtrLikeOptional(zcu)) {2133 // stores as := opcode, offset, alignment (opcode::memarg)
2314 return cg.store(lhs, rhs, Type.usize, offset);2134 try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
2315 }2135 @intFromEnum(std.wasm.SimdOpcode.v128_store),
2316 const pl_ty = ty.optionalChild(zcu);2136 offset + lhs.offset(),
2317 if (!pl_ty.hasRuntimeBits(zcu)) {2137 @intCast(ty.abiAlignment(zcu).toByteUnits() orelse 0),
2318 return cg.store(lhs, rhs, Type.u8, offset);2138 });
2319 }2139 return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2320 if (pl_ty.zigTypeTag(zcu) == .error_set) {2140 }
2321 return cg.store(lhs, rhs, Type.anyerror, offset);
2322 }
23232141
2324 const len = @as(u32, @intCast(abi_size));2142 const store_opcode: Mir.Inst.Tag = opcode: {
2325 assert(offset == 0);2143 if (ty.isAnyFloat()) {
2326 return cg.memcpy(lhs, rhs, .{ .imm32 = len });2144 break :opcode switch (abi_size) {
2327 },2145 2 => .i32_store16,
2328 .@"struct", .array, .@"union" => if (isByRef(ty, zcu, cg.target)) {2146 4 => .f32_store,
2329 const len = @as(u32, @intCast(abi_size));2147 8 => .f64_store,
2330 assert(offset == 0);2148 else => unreachable,
2331 return cg.memcpy(lhs, rhs, .{ .imm32 = len });2149 };
2332 },2150 } else {
2333 .vector => switch (determineSimdStoreStrategy(ty, zcu, cg.target)) {2151 break :opcode switch (abi_size) {
2334 .unrolled => {2152 1 => .i32_store8,
2335 const len: u32 = @intCast(abi_size);2153 2 => .i32_store16,
2336 return cg.memcpy(lhs, rhs, .{ .imm32 = len });2154 4 => .i32_store,
2337 },2155 8 => .i64_store,
2338 .direct => {2156 else => unreachable,
2339 try cg.emitWValue(lhs);2157 };
2340 try cg.lowerToStack(rhs);2158 }
2341 // TODO: Add helper functions for simd opcodes2159 };
2342 const extra_index: u32 = @intCast(cg.mir_extra.items.len);
2343 // stores as := opcode, offset, alignment (opcode::memarg)
2344 try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
2345 @intFromEnum(std.wasm.SimdOpcode.v128_store),
2346 offset + lhs.offset(),
2347 @intCast(ty.abiAlignment(zcu).toByteUnits() orelse 0),
2348 });
2349 return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2350 },
2351 },
2352 .pointer => {
2353 if (ty.isSlice(zcu)) {
2354 assert(offset == 0);
2355 // store pointer first
2356 // lower it to the stack so we do not have to store rhs into a local first
2357 try cg.emitWValue(lhs);
2358 const ptr_local = try cg.load(rhs, Type.usize, 0);
2359 try cg.store(.stack, ptr_local, Type.usize, 0 + lhs.offset());
2360
2361 // retrieve length from rhs, and store that alongside lhs as well
2362 try cg.emitWValue(lhs);
2363 const len_local = try cg.load(rhs, Type.usize, cg.ptrSize());
2364 try cg.store(.stack, len_local, Type.usize, cg.ptrSize() + lhs.offset());
2365 return;
2366 }
2367 },
2368 .int, .@"enum", .float => if (abi_size > 8 and abi_size <= 16) {
2369 assert(offset == 0);
2370 try cg.emitWValue(lhs);
2371 const lsb = try cg.load(rhs, Type.u64, 0);
2372 try cg.store(.stack, lsb, Type.u64, 0 + lhs.offset());
23732160
2374 try cg.emitWValue(lhs);
2375 const msb = try cg.load(rhs, Type.u64, 8);
2376 try cg.store(.stack, msb, Type.u64, 8 + lhs.offset());
2377 return;
2378 } else if (abi_size > 16) {
2379 assert(offset == 0);
2380 try cg.memcpy(lhs, rhs, .{ .imm32 = @as(u32, @intCast(ty.abiSize(zcu))) });
2381 },
2382 else => if (abi_size > 8) {
2383 return cg.fail("TODO: `store` for type `{f}` with abisize `{d}`", .{ ty.fmt(pt), abi_size });
2384 },
2385 }
2386 try cg.emitWValue(lhs);2161 try cg.emitWValue(lhs);
2387 // In this case we're actually interested in storing the stack position
2388 // into lhs, so we calculate that and emit that instead
2389 try cg.lowerToStack(rhs);2162 try cg.lowerToStack(rhs);
23902163
2391 const valtype = typeToValtype(ty, zcu, cg.target);
2392 const opcode = buildOpcode(.{
2393 .valtype1 = valtype,
2394 .width = @as(u8, @intCast(abi_size * 8)),
2395 .op = .store,
2396 });
2397
2398 // store rhs value at stack pointer's location in memory
2399 try cg.addMemArg(2164 try cg.addMemArg(
2400 Mir.Inst.Tag.fromOpcode(opcode),2165 store_opcode,
2401 .{2166 .{
2402 .offset = offset + lhs.offset(),2167 .offset = offset + lhs.offset(),
2403 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),2168 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
...@@ -2416,6 +2181,8 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2416,6 +2181,8 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
24162181
2417 if (!ty.hasRuntimeBits(zcu)) return cg.finishAir(inst, .none, &.{ty_op.operand});2182 if (!ty.hasRuntimeBits(zcu)) return cg.finishAir(inst, .none, &.{ty_op.operand});
24182183
2184 assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_load
2185
2419 const result = result: {2186 const result = result: {
2420 if (isByRef(ty, zcu, cg.target)) {2187 if (isByRef(ty, zcu, cg.target)) {
2421 const new_local = try cg.allocStack(ty);2188 const new_local = try cg.allocStack(ty);
...@@ -2423,30 +2190,17 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2423,30 +2190,17 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2423 break :result new_local;2190 break :result new_local;
2424 }2191 }
24252192
2426 if (ptr_info.packed_offset.host_size == 0) {2193 const loaded = try cg.load(operand, ty, 0);
2427 const loaded = try cg.load(operand, ty, 0);2194 const ty_size = ty.abiSize(zcu);
2428 const ty_size = ty.abiSize(zcu);2195 if (ty.isAbiInt(zcu) and ty_size * 8 > ty.bitSize(zcu)) {
2429 if (ty.isAbiInt(zcu) and ty_size * 8 > ty.bitSize(zcu)) {2196 const int_info = ty.intInfo(zcu);
2430 const int_elem_ty = try pt.intType(.unsigned, @intCast(ty_size * 8));2197 const loaded_int_ty: IntType = .{
2431 break :result try cg.trunc(loaded, ty, int_elem_ty);2198 .is_signed = int_info.signedness == .signed,
2432 } else {2199 .bits = @intCast(ty_size * 8),
2433 break :result loaded;2200 };
2434 }2201 break :result try cg.intTrunc(.fromType(cg, ty), loaded_int_ty, loaded);
2435 } else {2202 } else {
2436 const int_elem_ty = try pt.intType(.unsigned, ptr_info.packed_offset.host_size * 8);2203 break :result loaded;
2437 const shift_val: WValue = if (ptr_info.packed_offset.host_size <= 4)
2438 .{ .imm32 = ptr_info.packed_offset.bit_offset }
2439 else if (ptr_info.packed_offset.host_size <= 8)
2440 .{ .imm64 = ptr_info.packed_offset.bit_offset }
2441 else
2442 .{ .imm32 = ptr_info.packed_offset.bit_offset };
2443
2444 const stack_loaded = if (ptr_info.packed_offset.host_size <= 8)
2445 try cg.load(operand, int_elem_ty, 0)
2446 else
2447 operand;
2448 const shifted = try cg.binOp(stack_loaded, shift_val, int_elem_ty, .shr);
2449 break :result try cg.trunc(shifted, ty, int_elem_ty);
2450 }2204 }
2451 };2205 };
2452 return cg.finishAir(inst, result, &.{ty_op.operand});2206 return cg.finishAir(inst, result, &.{ty_op.operand});
...@@ -2472,20 +2226,33 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue...@@ -2472,20 +2226,33 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue
2472 return .stack;2226 return .stack;
2473 }2227 }
24742228
2475 const abi_size: u8 = @intCast(ty.abiSize(zcu));2229 const abi_size = ty.abiSize(zcu);
2476 const opcode = buildOpcode(.{2230 const load_opcode: Mir.Inst.Tag = opcode: {
2477 .valtype1 = typeToValtype(ty, zcu, cg.target),2231 if (ty.isAnyFloat()) {
2478 .width = abi_size * 8,2232 break :opcode switch (abi_size) {
2479 .op = .load,2233 2 => .i32_load16_u,
2480 .signedness = if (ty.isSignedInt(zcu)) .signed else .unsigned,2234 4 => .f32_load,
2481 });2235 8 => .f64_load,
24822236 else => unreachable,
2483 try cg.addMemArg(2237 };
2484 Mir.Inst.Tag.fromOpcode(opcode),2238 } else {
2485 .{2239 const is_signed = if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness == .signed else false;
2486 .offset = offset + operand.offset(),2240 break :opcode switch (abi_size) {
2487 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),2241 1 => if (is_signed) .i32_load8_s else .i32_load8_u,
2488 },2242 2 => if (is_signed) .i32_load16_s else .i32_load16_u,
2243 4 => .i32_load,
2244 8 => .i64_load,
2245 else => unreachable,
2246 };
2247 }
2248 };
2249
2250 try cg.addMemArg(
2251 load_opcode,
2252 .{
2253 .offset = offset + operand.offset(),
2254 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
2255 },
2489 );2256 );
24902257
2491 return .stack;2258 return .stack;
...@@ -2518,4487 +2285,4354 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2518,4487 +2285,4354 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2518 return cg.finishAir(inst, arg, &.{});2285 return cg.finishAir(inst, arg, &.{});
2519}2286}
25202287
2521fn airBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: Op) InnerError!void {2288const IntType = struct {
2522 const zcu = cg.pt.zcu;2289 is_signed: bool,
2523 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;2290 bits: u16,
2524 const lhs = try cg.resolveInst(bin_op.lhs);2291
2525 const rhs = try cg.resolveInst(bin_op.rhs);2292 const @"i32": IntType = .{ .is_signed = true, .bits = 32 };
2526 const lhs_ty = cg.typeOf(bin_op.lhs);2293 const @"i64": IntType = .{ .is_signed = true, .bits = 64 };
2527 const rhs_ty = cg.typeOf(bin_op.rhs);2294 const @"u32": IntType = .{ .is_signed = false, .bits = 32 };
25282295 const @"u64": IntType = .{ .is_signed = false, .bits = 64 };
2529 // For certain operations, such as shifting, the types are different.2296
2530 // When converting this to a WebAssembly type, they *must* match to perform2297 // Adapted from x86_64 backend
2531 // an operation. For this reason we verify if the WebAssembly type is different, in which2298 // Differ from Type.intInfo as it treats pointers/booleans/packed/enums/errors as integer
2532 // case we first coerce the operands to the same type before performing the operation.2299 fn fromType(cg: *CodeGen, ty: Type) IntType {
2533 // For big integers we can ignore this as we will call into compiler-rt which handles this.2300 const zcu = cg.pt.zcu;
2534 const result = switch (op) {2301 const ip = &zcu.intern_pool;
2535 .shr, .shl => result: {2302 var ty_index = ty.ip_index;
2536 if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) {2303 while (true) switch (ip.indexToKey(ty_index)) {
2537 return cg.fail("TODO: implement vector '{s}' with scalar rhs", .{@tagName(op)});2304 .int_type => |int_type| return .{ .is_signed = int_type.signedness == .signed, .bits = int_type.bits },
2538 }2305 .ptr_type => |ptr_type| return switch (ptr_type.flags.size) {
2306 .one, .many, .c => .{ .is_signed = false, .bits = cg.target.ptrBitWidth() },
2307 .slice => unreachable,
2308 },
2309 .opt_type => |opt_child| return if (!Type.fromInterned(opt_child).hasRuntimeBits(zcu))
2310 .{ .is_signed = false, .bits = 1 }
2311 else switch (ip.indexToKey(opt_child)) {
2312 .ptr_type => |ptr_type| switch (ptr_type.flags.size) {
2313 .one, .many => switch (ptr_type.flags.is_allowzero) {
2314 false => .{ .is_signed = false, .bits = cg.target.ptrBitWidth() },
2315 true => unreachable,
2316 },
2317 .slice, .c => unreachable,
2318 },
2319 else => unreachable,
2320 },
2321 .error_union_type => |error_union_type| return if (!Type.fromInterned(error_union_type.payload_type)
2322 .hasRuntimeBits(zcu)) .{ .is_signed = false, .bits = zcu.errorSetBits() } else unreachable,
2323 .simple_type => |simple_type| return switch (simple_type) {
2324 .bool => .{ .is_signed = false, .bits = 1 },
2325 .anyerror => .{ .is_signed = false, .bits = zcu.errorSetBits() },
2326 .isize => .{ .is_signed = true, .bits = cg.target.ptrBitWidth() },
2327 .usize => .{ .is_signed = false, .bits = cg.target.ptrBitWidth() },
2328 .c_char => .{ .is_signed = cg.target.cCharSignedness() == .signed, .bits = cg.target.cTypeBitSize(.char) },
2329 .c_short => .{ .is_signed = true, .bits = cg.target.cTypeBitSize(.short) },
2330 .c_ushort => .{ .is_signed = false, .bits = cg.target.cTypeBitSize(.short) },
2331 .c_int => .{ .is_signed = true, .bits = cg.target.cTypeBitSize(.int) },
2332 .c_uint => .{ .is_signed = false, .bits = cg.target.cTypeBitSize(.int) },
2333 .c_long => .{ .is_signed = true, .bits = cg.target.cTypeBitSize(.long) },
2334 .c_ulong => .{ .is_signed = false, .bits = cg.target.cTypeBitSize(.long) },
2335 .c_longlong => .{ .is_signed = true, .bits = cg.target.cTypeBitSize(.longlong) },
2336 .c_ulonglong => .{ .is_signed = false, .bits = cg.target.cTypeBitSize(.longlong) },
2337 .f16, .f32, .f64, .f80, .f128, .c_longdouble => unreachable,
2338 .anyopaque, .void, .type, .comptime_int, .comptime_float, .noreturn, .null, .undefined, .enum_literal, .adhoc_inferred_error_set, .generic_poison => unreachable,
2339 },
2340 .struct_type => {
2341 const loaded_struct = ip.loadStructType(ty_index);
2342 switch (loaded_struct.layout) {
2343 .auto, .@"extern" => unreachable,
2344 .@"packed" => ty_index = loaded_struct.packed_backing_int_type,
2345 }
2346 },
2347 .union_type => return switch (ip.loadUnionType(ty_index).layout) {
2348 .auto, .@"extern" => unreachable,
2349 .@"packed" => .{ .is_signed = false, .bits = @intCast(ty.bitSize(zcu)) },
2350 },
2351 .enum_type => ty_index = ip.loadEnumType(ty_index).int_tag_type,
2352 .error_set_type, .inferred_error_set_type => return .{ .is_signed = false, .bits = zcu.errorSetBits() },
2353 else => unreachable,
2354 };
2355 }
2356};
25392357
2540 const lhs_wasm_bits = toWasmBits(@intCast(lhs_ty.bitSize(zcu))) orelse {2358fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2541 return cg.fail("TODO: implement '{s}' for types larger than 128 bits", .{@tagName(op)});2359 switch (ty.bits) {
2542 };2360 0 => unreachable,
2543 const rhs_wasm_bits = toWasmBits(@intCast(rhs_ty.bitSize(zcu))).?;2361 1...32 => {
2544 const new_rhs = if (lhs_wasm_bits != rhs_wasm_bits and lhs_wasm_bits != 128)2362 try cg.emitWValue(lhs);
2545 try (try cg.intcast(rhs, rhs_ty, lhs_ty)).toLocal(cg, lhs_ty)2363 try cg.emitWValue(rhs);
2546 else2364 try cg.addTag(.i32_add);
2547 rhs;2365 return .stack;
2548 break :result try cg.binOp(lhs, new_rhs, lhs_ty, op);
2549 },2366 },
2550 else => try cg.binOp(lhs, rhs, lhs_ty, op),2367 33...64 => {
2551 };2368 try cg.emitWValue(lhs);
2369 try cg.emitWValue(rhs);
2370 try cg.addTag(.i64_add);
2371 return .stack;
2372 },
2373 65...128 => {
2374 const result = try cg.allocStack(Type.u128);
25522375
2553 return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });2376 var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);
2554}2377 defer lhs_lsb.free(cg);
2378 var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);
2379 defer rhs_lsb.free(cg);
2380 var op_lsb = try (try cg.intAdd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2381 defer op_lsb.free(cg);
25552382
2556/// Performs a binary operation on the given `WValue`'s2383 const lhs_msb = try cg.load(lhs, Type.u64, 8);
2557/// NOTE: THis leaves the value on top of the stack.2384 const rhs_msb = try cg.load(rhs, Type.u64, 8);
2558fn binOp(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue {2385 const op_msb = try cg.intAdd(.u64, lhs_msb, rhs_msb);
2559 const pt = cg.pt;
2560 const zcu = pt.zcu;
2561 assert(!(lhs != .stack and rhs == .stack));
25622386
2563 if (ty.isAnyFloat()) {2387 const lt = try cg.intCmp(.u64, .lt, op_lsb, rhs_lsb);
2564 const float_op = FloatOp.fromOp(op);2388 const tmp = try cg.intCast(.u64, .u32, lt);
2565 return cg.floatOp(float_op, ty, &.{ lhs, rhs });2389 var tmp_op = try (try cg.intAdd(.u64, op_msb, tmp)).toLocal(cg, Type.u64);
2566 }2390 defer tmp_op.free(cg);
25672391
2568 if (isByRef(ty, zcu, cg.target)) {2392 try cg.store(result, op_lsb, Type.u64, 0);
2569 if (ty.zigTypeTag(zcu) == .int) {2393 try cg.store(result, tmp_op, Type.u64, 8);
2570 return cg.binOpBigInt(lhs, rhs, ty, op);2394 return result;
2571 } else {2395 },
2572 return cg.fail("TODO: Implement binary operation for type: {f}", .{ty.fmt(pt)});2396 else => return cg.fail("TODO: Support intAdd for integer bitsize: {d}", .{ty.bits}),
2573 }
2574 }2397 }
2575
2576 const opcode: std.wasm.Opcode = buildOpcode(.{
2577 .op = op,
2578 .valtype1 = typeToValtype(ty, zcu, cg.target),
2579 .signedness = if (ty.isSignedInt(zcu)) .signed else .unsigned,
2580 });
2581 try cg.emitWValue(lhs);
2582 try cg.emitWValue(rhs);
2583
2584 try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode));
2585
2586 return .stack;
2587}2398}
25882399
2589fn binOpBigInt(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue {2400fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2590 const zcu = cg.pt.zcu;2401 switch (ty.bits) {
2591 const int_info = ty.intInfo(zcu);2402 0 => unreachable,
2592 if (int_info.bits > 128) {2403 1...32 => {
2593 return cg.fail("TODO: Implement binary operation for big integers larger than 128 bits", .{});2404 try cg.emitWValue(lhs);
2594 }2405 try cg.emitWValue(rhs);
25952406 try cg.addTag(.i32_sub);
2596 switch (op) {2407 return .stack;
2597 .mul => return cg.callIntrinsic(.__multi3, &.{ ty.toIntern(), ty.toIntern() }, ty, &.{ lhs, rhs }),
2598 .div => switch (int_info.signedness) {
2599 .signed => return cg.callIntrinsic(.__divti3, &.{ ty.toIntern(), ty.toIntern() }, ty, &.{ lhs, rhs }),
2600 .unsigned => return cg.callIntrinsic(.__udivti3, &.{ ty.toIntern(), ty.toIntern() }, ty, &.{ lhs, rhs }),
2601 },
2602 .rem => switch (int_info.signedness) {
2603 .signed => return cg.callIntrinsic(.__modti3, &.{ ty.toIntern(), ty.toIntern() }, ty, &.{ lhs, rhs }),
2604 .unsigned => return cg.callIntrinsic(.__umodti3, &.{ ty.toIntern(), ty.toIntern() }, ty, &.{ lhs, rhs }),
2605 },2408 },
2606 .shr => switch (int_info.signedness) {2409 33...64 => {
2607 .signed => return cg.callIntrinsic(.__ashrti3, &.{ ty.toIntern(), .i32_type }, ty, &.{ lhs, rhs }),2410 try cg.emitWValue(lhs);
2608 .unsigned => return cg.callIntrinsic(.__lshrti3, &.{ ty.toIntern(), .i32_type }, ty, &.{ lhs, rhs }),2411 try cg.emitWValue(rhs);
2412 try cg.addTag(.i64_sub);
2413 return .stack;
2609 },2414 },
2610 .shl => return cg.callIntrinsic(.__ashlti3, &.{ ty.toIntern(), .i32_type }, ty, &.{ lhs, rhs }),2415 65...128 => {
2611 .@"and", .@"or", .xor => {2416 const result = try cg.allocStack(Type.u128);
2612 const result = try cg.allocStack(ty);
2613 try cg.emitWValue(result);
2614 const lhs_lsb = try cg.load(lhs, Type.u64, 0);
2615 const rhs_lsb = try cg.load(rhs, Type.u64, 0);
2616 const op_lsb = try cg.binOp(lhs_lsb, rhs_lsb, Type.u64, op);
2617 try cg.store(.stack, op_lsb, Type.u64, result.offset());
26182417
2619 try cg.emitWValue(result);
2620 const lhs_msb = try cg.load(lhs, Type.u64, 8);
2621 const rhs_msb = try cg.load(rhs, Type.u64, 8);
2622 const op_msb = try cg.binOp(lhs_msb, rhs_msb, Type.u64, op);
2623 try cg.store(.stack, op_msb, Type.u64, result.offset() + 8);
2624 return result;
2625 },
2626 .add, .sub => {
2627 const result = try cg.allocStack(ty);
2628 var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);2418 var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);
2629 defer lhs_lsb.free(cg);2419 defer lhs_lsb.free(cg);
2630 var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);2420 var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);
2631 defer rhs_lsb.free(cg);2421 defer rhs_lsb.free(cg);
2632 var op_lsb = try (try cg.binOp(lhs_lsb, rhs_lsb, Type.u64, op)).toLocal(cg, Type.u64);2422 var op_lsb = try (try cg.intSub(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2633 defer op_lsb.free(cg);2423 defer op_lsb.free(cg);
26342424
2635 const lhs_msb = try cg.load(lhs, Type.u64, 8);2425 const lhs_msb = try cg.load(lhs, Type.u64, 8);
2636 const rhs_msb = try cg.load(rhs, Type.u64, 8);2426 const rhs_msb = try cg.load(rhs, Type.u64, 8);
2637 const op_msb = try cg.binOp(lhs_msb, rhs_msb, Type.u64, op);2427 const op_msb = try cg.intSub(.u64, lhs_msb, rhs_msb);
26382428
2639 const lt = if (op == .add) blk: {2429 const lt = try cg.intCmp(.u64, .lt, lhs_lsb, rhs_lsb);
2640 break :blk try cg.cmp(op_lsb, rhs_lsb, Type.u64, .lt);2430 const tmp = try cg.intCast(.u64, .u32, lt);
2641 } else if (op == .sub) blk: {2431 var tmp_op = try (try cg.intSub(.u64, op_msb, tmp)).toLocal(cg, Type.u64);
2642 break :blk try cg.cmp(lhs_lsb, rhs_lsb, Type.u64, .lt);
2643 } else unreachable;
2644 const tmp = try cg.intcast(lt, Type.u32, Type.u64);
2645 var tmp_op = try (try cg.binOp(op_msb, tmp, Type.u64, op)).toLocal(cg, Type.u64);
2646 defer tmp_op.free(cg);2432 defer tmp_op.free(cg);
26472433
2648 try cg.store(result, op_lsb, Type.u64, 0);2434 try cg.store(result, op_lsb, Type.u64, 0);
2649 try cg.store(result, tmp_op, Type.u64, 8);2435 try cg.store(result, tmp_op, Type.u64, 8);
2650 return result;2436 return result;
2651 },2437 },
2652 else => return cg.fail("TODO: Implement binary operation for big integers: '{s}'", .{@tagName(op)}),2438 else => return cg.fail("TODO: Support intSub for integer bitsize: {d}", .{ty.bits}),
2653 }
2654}
2655
2656const FloatOp = enum {
2657 add,
2658 ceil,
2659 cos,
2660 div,
2661 exp,
2662 exp2,
2663 fabs,
2664 floor,
2665 fma,
2666 fmax,
2667 fmin,
2668 fmod,
2669 log,
2670 log10,
2671 log2,
2672 mul,
2673 neg,
2674 round,
2675 sin,
2676 sqrt,
2677 sub,
2678 tan,
2679 trunc,
2680
2681 pub fn fromOp(op: Op) FloatOp {
2682 return switch (op) {
2683 .add => .add,
2684 .ceil => .ceil,
2685 .div => .div,
2686 .abs => .fabs,
2687 .floor => .floor,
2688 .max => .fmax,
2689 .min => .fmin,
2690 .mul => .mul,
2691 .neg => .neg,
2692 .nearest => .round,
2693 .sqrt => .sqrt,
2694 .sub => .sub,
2695 .trunc => .trunc,
2696 .rem => .fmod,
2697 else => unreachable,
2698 };
2699 }2439 }
2440}
27002441
2701 pub fn toOp(float_op: FloatOp) ?Op {2442fn intMul(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2702 return switch (float_op) {2443 switch (ty.bits) {
2703 .add => .add,2444 0 => unreachable,
2704 .ceil => .ceil,2445 1...32 => {
2705 .div => .div,2446 try cg.emitWValue(lhs);
2706 .fabs => .abs,2447 try cg.emitWValue(rhs);
2707 .floor => .floor,2448 try cg.addTag(.i32_mul);
2708 .fmax => .max,2449 return .stack;
2709 .fmin => .min,2450 },
2710 .mul => .mul,2451 33...64 => {
2711 .neg => .neg,2452 try cg.emitWValue(lhs);
2712 .round => .nearest,2453 try cg.emitWValue(rhs);
2713 .sqrt => .sqrt,2454 try cg.addTag(.i64_mul);
2714 .sub => .sub,2455 return .stack;
2715 .trunc => .trunc,2456 },
27162457 65...128 => return cg.callIntrinsic(.__multi3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs }),
2717 .cos,2458 else => return cg.fail("TODO: Support intMul for integer bitsize: {d}", .{ty.bits}),
2718 .exp,
2719 .exp2,
2720 .fma,
2721 .fmod,
2722 .log,
2723 .log10,
2724 .log2,
2725 .sin,
2726 .tan,
2727 => null,
2728 };
2729 }2459 }
2460}
27302461
2731 fn intrinsic(op: FloatOp, bits: u16) Mir.Intrinsic {2462fn intDiv(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2732 return switch (op) {2463 switch (ty.bits) {
2733 inline .add, .sub, .div, .mul => |ct_op| switch (bits) {2464 0 => unreachable,
2734 inline 16, 80, 128 => |ct_bits| @field(2465 1...32 => {
2735 Mir.Intrinsic,2466 try cg.emitWValue(lhs);
2736 "__" ++ @tagName(ct_op) ++ compilerRtFloatAbbrev(ct_bits) ++ "f3",2467 try cg.emitWValue(rhs);
2737 ),2468 try cg.addTag(if (ty.is_signed) .i32_div_s else .i32_div_u);
2738 else => unreachable,2469 return .stack;
2739 },2470 },
27402471 33...64 => {
2741 inline .ceil,2472 try cg.emitWValue(lhs);
2742 .fabs,2473 try cg.emitWValue(rhs);
2743 .floor,2474 try cg.addTag(if (ty.is_signed) .i64_div_s else .i64_div_u);
2744 .fmax,2475 return .stack;
2745 .fmin,2476 },
2746 .round,2477 65...128 => {
2747 .sqrt,2478 if (ty.is_signed) {
2748 .trunc,2479 return cg.callIntrinsic(.__divti3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs });
2749 => |ct_op| switch (bits) {2480 } else {
2750 inline 16, 80, 128 => |ct_bits| @field(2481 return cg.callIntrinsic(.__udivti3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs });
2751 Mir.Intrinsic,2482 }
2752 libcFloatPrefix(ct_bits) ++ @tagName(ct_op) ++ libcFloatSuffix(ct_bits),2483 },
2753 ),2484 else => return cg.fail("TODO: Support intDiv for integer bitsize: {d}", .{ty.bits}),
2754 else => unreachable,2485 }
2755 },2486}
2756
2757 inline .cos,
2758 .exp,
2759 .exp2,
2760 .fma,
2761 .fmod,
2762 .log,
2763 .log10,
2764 .log2,
2765 .sin,
2766 .tan,
2767 => |ct_op| switch (bits) {
2768 inline 16, 32, 64, 80, 128 => |ct_bits| @field(
2769 Mir.Intrinsic,
2770 libcFloatPrefix(ct_bits) ++ @tagName(ct_op) ++ libcFloatSuffix(ct_bits),
2771 ),
2772 else => unreachable,
2773 },
27742487
2775 .neg => unreachable,2488fn intDivFloor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2776 };2489 if (!ty.is_signed) {
2490 return cg.intDiv(ty, lhs, rhs);
2777 }2491 }
2778};
27792492
2780fn airAbs(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {2493 switch (ty.bits) {
2781 const pt = cg.pt;2494 0 => unreachable,
2782 const zcu = pt.zcu;2495 1...32 => {
2783 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;2496 var q = try (try cg.intDiv(ty, lhs, rhs)).toLocal(cg, Type.i32);
2784 const operand = try cg.resolveInst(ty_op.operand);2497 defer q.free(cg);
2785 const ty = cg.typeOf(ty_op.operand);
2786 const scalar_ty = ty.scalarType(zcu);
27872498
2788 switch (scalar_ty.zigTypeTag(zcu)) {2499 const zero: WValue = .{ .imm32 = 0 };
2789 .int => if (ty.zigTypeTag(zcu) == .vector) {
2790 return cg.fail("TODO implement airAbs for {f}", .{ty.fmt(pt)});
2791 } else {
2792 const int_bits = ty.intInfo(zcu).bits;
2793 const wasm_bits = toWasmBits(int_bits) orelse {
2794 return cg.fail("TODO: airAbs for signed integers larger than '{d}' bits", .{int_bits});
2795 };
27962500
2797 switch (wasm_bits) {2501 const r = try cg.intRem(ty, lhs, rhs);
2798 32 => {2502 var r_nonzero = try (try cg.intCmp(ty, .neq, r, zero)).toLocal(cg, Type.i32);
2799 try cg.emitWValue(operand);2503 defer r_nonzero.free(cg);
28002504
2801 try cg.addImm32(31);2505 const sign_xor = try cg.intXor(ty, lhs, rhs);
2802 try cg.addTag(.i32_shr_s);2506 var sign_diff = try (try cg.intCmp(ty, .lt, sign_xor, zero)).toLocal(cg, Type.i32);
2507 defer sign_diff.free(cg);
28032508
2804 var tmp = try cg.allocLocal(ty);2509 try cg.emitWValue(q);
2805 defer tmp.free(cg);2510 const need_adjust = try cg.intAnd(.u32, r_nonzero, sign_diff);
2806 try cg.addLocal(.local_tee, tmp.local.value);2511 try cg.emitWValue(need_adjust);
2512 try cg.addTag(.i32_sub);
2513 return .stack;
2514 },
2515 33...64 => {
2516 var q = try (try cg.intDiv(ty, lhs, rhs)).toLocal(cg, Type.i64);
2517 defer q.free(cg);
28072518
2808 try cg.emitWValue(operand);2519 const zero: WValue = .{ .imm64 = 0 };
2809 try cg.addTag(.i32_xor);
2810 try cg.emitWValue(tmp);
2811 try cg.addTag(.i32_sub);
2812 return cg.finishAir(inst, .stack, &.{ty_op.operand});
2813 },
2814 64 => {
2815 try cg.emitWValue(operand);
28162520
2817 try cg.addImm64(63);2521 const r = try cg.intRem(ty, lhs, rhs);
2818 try cg.addTag(.i64_shr_s);2522 var r_nonzero = try (try cg.intCmp(ty, .neq, r, zero)).toLocal(cg, Type.i32);
2523 defer r_nonzero.free(cg);
28192524
2820 var tmp = try cg.allocLocal(ty);2525 const sign_xor = try cg.intXor(ty, lhs, rhs);
2821 defer tmp.free(cg);2526 var sign_diff = try (try cg.intCmp(ty, .lt, sign_xor, zero)).toLocal(cg, Type.i32);
2822 try cg.addLocal(.local_tee, tmp.local.value);2527 defer sign_diff.free(cg);
28232528
2824 try cg.emitWValue(operand);2529 try cg.emitWValue(q);
2825 try cg.addTag(.i64_xor);2530 const need_adjust = try cg.intAnd(.u32, r_nonzero, sign_diff);
2826 try cg.emitWValue(tmp);2531 try cg.emitWValue(need_adjust);
2827 try cg.addTag(.i64_sub);2532 try cg.addTag(.i64_extend_i32_u);
2828 return cg.finishAir(inst, .stack, &.{ty_op.operand});2533 try cg.addTag(.i64_sub);
2829 },2534 return .stack;
2830 128 => {
2831 const mask = try cg.allocStack(Type.u128);
2832 try cg.emitWValue(mask);
2833 try cg.emitWValue(mask);
2834
2835 _ = try cg.load(operand, Type.u64, 8);
2836 try cg.addImm64(63);
2837 try cg.addTag(.i64_shr_s);
2838
2839 var tmp = try cg.allocLocal(Type.u64);
2840 defer tmp.free(cg);
2841 try cg.addLocal(.local_tee, tmp.local.value);
2842 try cg.store(.stack, .stack, Type.u64, mask.offset() + 0);
2843 try cg.emitWValue(tmp);
2844 try cg.store(.stack, .stack, Type.u64, mask.offset() + 8);
2845
2846 const a = try cg.binOpBigInt(operand, mask, Type.u128, .xor);
2847 const b = try cg.binOpBigInt(a, mask, Type.u128, .sub);
2848
2849 return cg.finishAir(inst, b, &.{ty_op.operand});
2850 },
2851 else => unreachable,
2852 }
2853 },
2854 .float => {
2855 const result = try cg.floatOp(.fabs, ty, &.{operand});
2856 return cg.finishAir(inst, result, &.{ty_op.operand});
2857 },2535 },
2858 else => unreachable,2536 else => return cg.fail("TODO: Support intDivFloor for signed integer bitsize: {d}", .{ty.bits}),
2859 }2537 }
2860}2538}
28612539
2862fn airUnaryFloatOp(cg: *CodeGen, inst: Air.Inst.Index, op: FloatOp) InnerError!void {2540fn intRem(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2863 const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;2541 switch (ty.bits) {
2864 const operand = try cg.resolveInst(un_op);2542 0 => unreachable,
2865 const ty = cg.typeOf(un_op);2543 1...32 => {
28662544 try cg.emitWValue(lhs);
2867 const result = try cg.floatOp(op, ty, &.{operand});2545 try cg.emitWValue(rhs);
2868 return cg.finishAir(inst, result, &.{un_op});2546 try cg.addTag(if (ty.is_signed) .i32_rem_s else .i32_rem_u);
2547 return .stack;
2548 },
2549 33...64 => {
2550 try cg.emitWValue(lhs);
2551 try cg.emitWValue(rhs);
2552 try cg.addTag(if (ty.is_signed) .i64_rem_s else .i64_rem_u);
2553 return .stack;
2554 },
2555 65...128 => {
2556 if (ty.is_signed) {
2557 return cg.callIntrinsic(.__modti3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs });
2558 } else {
2559 return cg.callIntrinsic(.__umodti3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs });
2560 }
2561 },
2562 else => return cg.fail("TODO: Support intRem for integer bitsize: {d}", .{ty.bits}),
2563 }
2869}2564}
28702565
2871fn floatOp(cg: *CodeGen, float_op: FloatOp, ty: Type, args: []const WValue) InnerError!WValue {2566fn intMod(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2872 const zcu = cg.pt.zcu;2567 if (!ty.is_signed) {
2873 if (ty.zigTypeTag(zcu) == .vector) {2568 return cg.intRem(ty, lhs, rhs);
2874 return cg.fail("TODO: Implement floatOps for vectors", .{});
2875 }2569 }
28762570
2877 const float_bits = ty.floatBits(cg.target);2571 // mod_s(a, b) = rem_s(rem_s(a, b) + b, b)
28782572 const rem = try cg.intRem(ty, lhs, rhs);
2879 if (float_op == .neg) {2573 const sum = try cg.intAdd(ty, rem, rhs);
2880 return cg.floatNeg(ty, args[0]);2574 return cg.intRem(ty, sum, rhs);
2881 }2575}
28822576
2883 if (float_bits == 32 or float_bits == 64) {2577fn intAnd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2884 if (float_op.toOp()) |op| {2578 switch (ty.bits) {
2885 for (args) |operand| {2579 0 => unreachable,
2886 try cg.emitWValue(operand);2580 1...32 => {
2887 }2581 try cg.emitWValue(lhs);
2888 const opcode = buildOpcode(.{ .op = op, .valtype1 = typeToValtype(ty, zcu, cg.target) });2582 try cg.emitWValue(rhs);
2889 try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode));2583 try cg.addTag(.i32_and);
2890 return .stack;2584 return .stack;
2891 }2585 },
2892 }2586 33...64 => {
2587 try cg.emitWValue(lhs);
2588 try cg.emitWValue(rhs);
2589 try cg.addTag(.i64_and);
2590 return .stack;
2591 },
2592 65...128 => {
2593 const result = try cg.allocStack(Type.u128);
2594
2595 const lhs_lsb = try cg.load(lhs, Type.u64, 0);
2596 const rhs_lsb = try cg.load(rhs, Type.u64, 0);
2597 const and_lsb = try (try cg.intAnd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2598 try cg.store(result, and_lsb, Type.u64, 0);
28932599
2894 const intrinsic = float_op.intrinsic(float_bits);2600 const lhs_msb = try cg.load(lhs, Type.u64, 8);
2601 const rhs_msb = try cg.load(rhs, Type.u64, 8);
2602 const and_msb = try (try cg.intAnd(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
2603 try cg.store(result, and_msb, Type.u64, 8);
28952604
2896 // fma requires three operands2605 return result;
2897 var param_types_buffer: [3]InternPool.Index = .{ ty.ip_index, ty.ip_index, ty.ip_index };2606 },
2898 const param_types = param_types_buffer[0..args.len];2607 else => return cg.fail("TODO: Support intAnd for integer bitsize: {d}", .{ty.bits}),
2899 return cg.callIntrinsic(intrinsic, param_types, ty, args);2608 }
2900}2609}
29012610
2902/// NOTE: The result value remains on top of the stack.2611fn intOr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2903fn floatNeg(cg: *CodeGen, ty: Type, arg: WValue) InnerError!WValue {2612 switch (ty.bits) {
2904 const float_bits = ty.floatBits(cg.target);2613 0 => unreachable,
2905 switch (float_bits) {2614 1...32 => {
2906 16 => {2615 try cg.emitWValue(lhs);
2907 try cg.emitWValue(arg);2616 try cg.emitWValue(rhs);
2908 try cg.addImm32(0x8000);2617 try cg.addTag(.i32_or);
2909 try cg.addTag(.i32_xor);
2910 return .stack;2618 return .stack;
2911 },2619 },
2912 32, 64 => {2620 33...64 => {
2913 try cg.emitWValue(arg);2621 try cg.emitWValue(lhs);
2914 const val_type: std.wasm.Valtype = if (float_bits == 32) .f32 else .f64;2622 try cg.emitWValue(rhs);
2915 const opcode = buildOpcode(.{ .op = .neg, .valtype1 = val_type });2623 try cg.addTag(.i64_or);
2916 try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode));
2917 return .stack;2624 return .stack;
2918 },2625 },
2919 80, 128 => {2626 65...128 => {
2920 const result = try cg.allocStack(ty);2627 const result = try cg.allocStack(Type.u128);
2921 try cg.emitWValue(result);
2922 try cg.emitWValue(arg);
2923 try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 });
2924 try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 });
29252628
2926 try cg.emitWValue(result);2629 const lhs_lsb = try cg.load(lhs, Type.u64, 0);
2927 try cg.emitWValue(arg);2630 const rhs_lsb = try cg.load(rhs, Type.u64, 0);
2928 try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 });2631 const or_lsb = try (try cg.intOr(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2632 try cg.store(result, or_lsb, Type.u64, 0);
2633
2634 const lhs_msb = try cg.load(lhs, Type.u64, 8);
2635 const rhs_msb = try cg.load(rhs, Type.u64, 8);
2636 const or_msb = try (try cg.intOr(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
2637 try cg.store(result, or_msb, Type.u64, 8);
29292638
2930 if (float_bits == 80) {
2931 try cg.addImm64(0x8000);
2932 try cg.addTag(.i64_xor);
2933 try cg.addMemArg(.i64_store16, .{ .offset = 8 + result.offset(), .alignment = 2 });
2934 } else {
2935 try cg.addImm64(0x8000000000000000);
2936 try cg.addTag(.i64_xor);
2937 try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 });
2938 }
2939 return result;2639 return result;
2940 },2640 },
2941 else => unreachable,2641 else => return cg.fail("TODO: Support intOr for integer bitsize: {d}", .{ty.bits}),
2942 }2642 }
2943}2643}
29442644
2945fn airWrapBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: Op) InnerError!void {2645fn intXor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2946 const zcu = cg.pt.zcu;2646 switch (ty.bits) {
2947 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;2647 0 => unreachable,
2648 1...32 => {
2649 try cg.emitWValue(lhs);
2650 try cg.emitWValue(rhs);
2651 try cg.addTag(.i32_xor);
2652 return .stack;
2653 },
2654 33...64 => {
2655 try cg.emitWValue(lhs);
2656 try cg.emitWValue(rhs);
2657 try cg.addTag(.i64_xor);
2658 return .stack;
2659 },
2660 65...128 => {
2661 const result = try cg.allocStack(Type.u128);
29482662
2949 const lhs = try cg.resolveInst(bin_op.lhs);2663 const lhs_lsb = try cg.load(lhs, Type.u64, 0);
2950 const rhs = try cg.resolveInst(bin_op.rhs);2664 const rhs_lsb = try cg.load(rhs, Type.u64, 0);
2951 const lhs_ty = cg.typeOf(bin_op.lhs);2665 const xor_lsb = try (try cg.intXor(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2952 const rhs_ty = cg.typeOf(bin_op.rhs);2666 try cg.store(result, xor_lsb, Type.u64, 0);
29532667
2954 if (lhs_ty.isVector(zcu)) {2668 const lhs_msb = try cg.load(lhs, Type.u64, 8);
2955 if ((op == .shr or op == .shl) and !rhs_ty.isVector(zcu)) {2669 const rhs_msb = try cg.load(rhs, Type.u64, 8);
2956 return cg.fail("TODO: implement wrapping vector '{s}' with scalar rhs", .{@tagName(op)});2670 const xor_msb = try (try cg.intXor(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
2957 } else {2671 try cg.store(result, xor_msb, Type.u64, 8);
2958 return cg.fail("TODO: implement wrapping '{s}' for vectors", .{@tagName(op)});
2959 }
2960 }
29612672
2962 // For certain operations, such as shifting, the types are different.2673 return result;
2963 // When converting this to a WebAssembly type, they *must* match to perform
2964 // an operation. For this reason we verify if the WebAssembly type is different, in which
2965 // case we first coerce the operands to the same type before performing the operation.
2966 // For big integers we can ignore this as we will call into compiler-rt which handles this.
2967 const result = switch (op) {
2968 .shr, .shl => result: {
2969 const lhs_wasm_bits = toWasmBits(@intCast(lhs_ty.bitSize(zcu))) orelse {
2970 return cg.fail("TODO: implement '{s}' for types larger than 128 bits", .{@tagName(op)});
2971 };
2972 const rhs_wasm_bits = toWasmBits(@intCast(rhs_ty.bitSize(zcu))).?;
2973 const new_rhs = if (lhs_wasm_bits != rhs_wasm_bits and lhs_wasm_bits != 128)
2974 try (try cg.intcast(rhs, rhs_ty, lhs_ty)).toLocal(cg, lhs_ty)
2975 else
2976 rhs;
2977 break :result try cg.wrapBinOp(lhs, new_rhs, lhs_ty, op);
2978 },2674 },
2979 else => try cg.wrapBinOp(lhs, rhs, lhs_ty, op),2675 else => return cg.fail("TODO: Support intXor for integer bitsize: {d}", .{ty.bits}),
2980 };2676 }
2981
2982 return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
2983}
2984
2985/// Performs a wrapping binary operation.
2986/// Asserts rhs is not a stack value when lhs also isn't.
2987/// NOTE: Leaves the result on the stack when its Type is <= 64 bits
2988fn wrapBinOp(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue {
2989 const bin_local = try cg.binOp(lhs, rhs, ty, op);
2990 return cg.wrapOperand(bin_local, ty);
2991}2677}
29922678
2993/// Wraps an operand based on a given type's bitsize.2679fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
2994/// Asserts `Type` is <= 128 bits.2680 switch (ty.bits) {
2995/// NOTE: When the Type is <= 64 bits, leaves the value on top of the stack, if wrapping was needed.2681 0 => unreachable,
2996fn wrapOperand(cg: *CodeGen, operand: WValue, ty: Type) InnerError!WValue {2682 1 => {
2997 const zcu = cg.pt.zcu;
2998 assert(ty.abiSize(zcu) <= 16);
2999 const int_bits: u16 = @intCast(ty.bitSize(zcu)); // TODO use ty.intInfo(zcu).bits
3000 const wasm_bits = toWasmBits(int_bits) orelse {
3001 return cg.fail("TODO: Implement wrapOperand for bitsize '{d}'", .{int_bits});
3002 };
3003
3004 if (wasm_bits == int_bits) return operand;
3005
3006 switch (wasm_bits) {
3007 32 => {
3008 try cg.emitWValue(operand);2683 try cg.emitWValue(operand);
3009 if (ty.isSignedInt(zcu)) {2684 if (ty.is_signed) {
3010 try cg.addImm32(32 - int_bits);2685 try cg.addImm32(~@as(u32, 0));
3011 try cg.addTag(.i32_shl);2686 try cg.addTag(.i32_xor);
3012 try cg.addImm32(32 - int_bits);
3013 try cg.addTag(.i32_shr_s);
3014 } else {2687 } else {
3015 try cg.addImm32(~@as(u32, 0) >> @intCast(32 - int_bits));2688 try cg.addTag(.i32_eqz);
3016 try cg.addTag(.i32_and);
3017 }2689 }
3018 return .stack;2690 return .stack;
3019 },2691 },
3020 64 => {2692 2...32 => {
2693 const mask: u32 = if (ty.is_signed)
2694 ~@as(u32, 0)
2695 else
2696 ~@as(u32, 0) >> @intCast(32 - ty.bits);
3021 try cg.emitWValue(operand);2697 try cg.emitWValue(operand);
3022 if (ty.isSignedInt(zcu)) {2698 try cg.addImm32(mask);
3023 try cg.addImm64(64 - int_bits);2699 try cg.addTag(.i32_xor);
3024 try cg.addTag(.i64_shl);
3025 try cg.addImm64(64 - int_bits);
3026 try cg.addTag(.i64_shr_s);
3027 } else {
3028 try cg.addImm64(~@as(u64, 0) >> @intCast(64 - int_bits));
3029 try cg.addTag(.i64_and);
3030 }
3031 return .stack;2700 return .stack;
3032 },2701 },
3033 128 => {2702 33...64 => {
3034 assert(operand != .stack);2703 const mask: u64 = if (ty.is_signed)
3035 const result = try cg.allocStack(ty);2704 ~@as(u64, 0)
30362705 else
3037 try cg.emitWValue(result);2706 ~@as(u64, 0) >> @intCast(64 - ty.bits);
2707 try cg.emitWValue(operand);
2708 try cg.addImm64(mask);
2709 try cg.addTag(.i64_xor);
2710 return .stack;
2711 },
2712 65...128 => {
2713 const result = try cg.allocStack(Type.u128);
2714
2715 try cg.emitWValue(result);
3038 _ = try cg.load(operand, Type.u64, 0);2716 _ = try cg.load(operand, Type.u64, 0);
2717 try cg.addImm64(~@as(u64, 0));
2718 try cg.addTag(.i64_xor);
3039 try cg.store(.stack, .stack, Type.u64, result.offset());2719 try cg.store(.stack, .stack, Type.u64, result.offset());
30402720
3041 try cg.emitWValue(result);2721 try cg.emitWValue(result);
3042 _ = try cg.load(operand, Type.u64, 8);2722 _ = try cg.load(operand, Type.u64, 8);
3043 if (ty.isSignedInt(zcu)) {2723 const high_mask: u64 = if (ty.is_signed)
3044 try cg.addImm64(128 - int_bits);2724 ~@as(u64, 0)
3045 try cg.addTag(.i64_shl);2725 else
3046 try cg.addImm64(128 - int_bits);2726 ~@as(u64, 0) >> @intCast(128 - ty.bits);
3047 try cg.addTag(.i64_shr_s);2727 try cg.addImm64(high_mask);
3048 } else {2728 try cg.addTag(.i64_xor);
3049 try cg.addImm64(~@as(u64, 0) >> @intCast(128 - int_bits));
3050 try cg.addTag(.i64_and);
3051 }
3052 try cg.store(.stack, .stack, Type.u64, result.offset() + 8);2729 try cg.store(.stack, .stack, Type.u64, result.offset() + 8);
30532730
3054 return result;2731 return result;
3055 },2732 },
3056 else => unreachable,2733 else => return cg.fail("TODO: Support intNot for integer bitsize: {d}", .{ty.bits}),
3057 }2734 }
3058}2735}
30592736
3060fn lowerPtr(cg: *CodeGen, ptr_val: InternPool.Index, prev_offset: u64) InnerError!WValue {2737// rhs is a shift count, pointing to i32 value
3061 const pt = cg.pt;2738fn intShl(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3062 const zcu = pt.zcu;2739 switch (ty.bits) {
3063 const ptr = zcu.intern_pool.indexToKey(ptr_val).ptr;2740 0 => unreachable,
3064 const offset: u64 = prev_offset + ptr.byte_offset;2741 1...32 => {
3065 return switch (ptr.base_addr) {2742 try cg.emitWValue(lhs);
3066 .nav => |nav| return .{ .nav_ref = .{ .nav_index = nav, .offset = @intCast(offset) } },2743 try cg.emitWValue(rhs);
3067 .uav => |uav| return .{ .uav_ref = .{ .ip_index = uav.val, .offset = @intCast(offset), .orig_ptr_ty = uav.orig_ty } },2744 try cg.addTag(.i32_shl);
3068 .int => return cg.lowerConstant(try pt.intValue(.usize, offset)),2745 return .stack;
3069 .eu_payload => |eu_ptr| try cg.lowerPtr(
3070 eu_ptr,
3071 offset + codegen.errUnionPayloadOffset(
3072 Value.fromInterned(eu_ptr).typeOf(zcu).childType(zcu),
3073 zcu,
3074 ),
3075 ),
3076 .opt_payload => |opt_ptr| return cg.lowerPtr(opt_ptr, offset),
3077 .field => |field| {
3078 const base_ptr = Value.fromInterned(field.base);
3079 const base_ty = base_ptr.typeOf(zcu).childType(zcu);
3080 const field_off: u64 = switch (base_ty.zigTypeTag(zcu)) {
3081 .pointer => off: {
3082 assert(base_ty.isSlice(zcu));
3083 break :off switch (field.index) {
3084 Value.slice_ptr_index => 0,
3085 Value.slice_len_index => @divExact(cg.target.ptrBitWidth(), 8),
3086 else => unreachable,
3087 };
3088 },
3089 .@"struct" => switch (base_ty.containerLayout(zcu)) {
3090 .auto => base_ty.structFieldOffset(@intCast(field.index), zcu),
3091 .@"extern", .@"packed" => unreachable,
3092 },
3093 .@"union" => switch (base_ty.containerLayout(zcu)) {
3094 .auto => base_ty.structFieldOffset(@intCast(field.index), zcu),
3095 .@"extern", .@"packed" => unreachable,
3096 },
3097 else => unreachable,
3098 };
3099 return cg.lowerPtr(field.base, offset + field_off);
3100 },
3101 .arr_elem, .comptime_field, .comptime_alloc => unreachable,
3102 };
3103}
3104
3105/// Asserts that `isByRef` returns `false` for `val.typeOf(zcu)`.
3106fn lowerConstant(cg: *CodeGen, val: Value) InnerError!WValue {
3107 const pt = cg.pt;
3108 const zcu = pt.zcu;
3109 const ty = val.typeOf(zcu);
3110 assert(!isByRef(ty, zcu, cg.target));
3111 const ip = &zcu.intern_pool;
3112 if (val.isUndef(zcu)) return cg.emitUndefined(ty);
3113
3114 switch (ip.indexToKey(val.ip_index)) {
3115 .int_type,
3116 .ptr_type,
3117 .array_type,
3118 .vector_type,
3119 .opt_type,
3120 .anyframe_type,
3121 .error_union_type,
3122 .simple_type,
3123 .struct_type,
3124 .tuple_type,
3125 .union_type,
3126 .opaque_type,
3127 .enum_type,
3128 .func_type,
3129 .error_set_type,
3130 .inferred_error_set_type,
3131 => unreachable, // types, not values
3132
3133 .undef => unreachable, // handled above
3134 .simple_value => |simple_value| switch (simple_value) {
3135 .void,
3136 .null,
3137 .@"unreachable",
3138 => unreachable, // non-runtime values
3139 .false, .true => return .{ .imm32 = switch (simple_value) {
3140 .false => 0,
3141 .true => 1,
3142 else => unreachable,
3143 } },
3144 },
3145 .variable,
3146 .@"extern",
3147 .func,
3148 .enum_literal,
3149 => unreachable, // non-runtime values
3150 .int => {
3151 const int_info = ty.intInfo(zcu);
3152 switch (int_info.signedness) {
3153 .signed => switch (int_info.bits) {
3154 0...32 => return .{ .imm32 = @bitCast(@as(i32, @intCast(val.toSignedInt(zcu)))) },
3155 33...64 => return .{ .imm64 = @bitCast(val.toSignedInt(zcu)) },
3156 else => unreachable,
3157 },
3158 .unsigned => switch (int_info.bits) {
3159 0...32 => return .{ .imm32 = @intCast(val.toUnsignedInt(zcu)) },
3160 33...64 => return .{ .imm64 = val.toUnsignedInt(zcu) },
3161 else => unreachable,
3162 },
3163 }
3164 },2746 },
3165 .err => |err| {2747 33...64 => {
3166 const int = try pt.getErrorValue(err.name);2748 try cg.emitWValue(lhs);
3167 return .{ .imm32 = int };2749 try cg.emitWValue(rhs);
2750 try cg.addTag(.i64_extend_i32_u);
2751 try cg.addTag(.i64_shl);
2752 return .stack;
3168 },2753 },
3169 .error_union => |error_union| {2754 65...128 => return cg.callIntrinsic(.__ashlti3, &.{ .i128_type, .i32_type }, Type.i128, &.{ lhs, rhs }),
3170 const err_int_ty = try pt.errorIntType();2755 else => return cg.fail("TODO: Support intShl for integer bitsize: {d}", .{ty.bits}),
3171 const err_val: Value = switch (error_union.val) {2756 }
3172 .err_name => |err_name| .fromInterned(try pt.intern(.{ .err = .{2757}
3173 .ty = ty.errorUnionSet(zcu).toIntern(),
3174 .name = err_name,
3175 } })),
3176 .payload => try pt.intValue(err_int_ty, 0),
3177 };
3178 const payload_type = ty.errorUnionPayload(zcu);
3179 if (!payload_type.hasRuntimeBits(zcu)) {
3180 // We use the error type directly as the type.
3181 return cg.lowerConstant(err_val);
3182 }
31832758
3184 return cg.fail("Wasm TODO: lowerConstant error union with non-zero-bit payload type", .{});2759// rhs is a shift count, pointing to i32 value
2760fn intShr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2761 switch (ty.bits) {
2762 0 => unreachable,
2763 1...32 => {
2764 try cg.emitWValue(lhs);
2765 try cg.emitWValue(rhs);
2766 try cg.addTag(if (ty.is_signed) .i32_shr_s else .i32_shr_u);
2767 return .stack;
3185 },2768 },
3186 .enum_tag => |enum_tag| return cg.lowerConstant(.fromInterned(enum_tag.int)),2769 33...64 => {
3187 .float => |float| switch (float.storage) {2770 try cg.emitWValue(lhs);
3188 .f16 => |f16_val| return .{ .imm32 = @as(u16, @bitCast(f16_val)) },2771 try cg.emitWValue(rhs);
3189 .f32 => |f32_val| return .{ .float32 = f32_val },2772 try cg.addTag(.i64_extend_i32_u);
3190 .f64 => |f64_val| return .{ .float64 = f64_val },2773 try cg.addTag(if (ty.is_signed) .i64_shr_s else .i64_shr_u);
3191 else => unreachable,2774 return .stack;
3192 },2775 },
3193 .slice => unreachable, // isByRef == true2776 65...128 => {
3194 .ptr => return cg.lowerPtr(val.toIntern(), 0),2777 if (ty.is_signed) {
3195 .opt => if (ty.optionalReprIsPayload(zcu)) {2778 return cg.callIntrinsic(.__ashrti3, &.{ .i128_type, .i32_type }, Type.i128, &.{ lhs, rhs });
3196 if (val.optionalValue(zcu)) |payload| {
3197 return cg.lowerConstant(payload);
3198 } else {2779 } else {
3199 return .{ .imm32 = 0 };2780 return cg.callIntrinsic(.__lshrti3, &.{ .i128_type, .i32_type }, Type.i128, &.{ lhs, rhs });
3200 }2781 }
3201 } else {
3202 return .{ .imm32 = @intFromBool(!val.isNull(zcu)) };
3203 },
3204 .aggregate => switch (ip.indexToKey(ty.ip_index)) {
3205 .array_type => return cg.fail("Wasm TODO: LowerConstant for {f}", .{ty.fmt(pt)}),
3206 .vector_type => {
3207 assert(determineSimdStoreStrategy(ty, zcu, cg.target) == .direct);
3208 var buf: [16]u8 = undefined;
3209 val.writeToMemory(pt, &buf) catch unreachable;
3210 return cg.storeSimdImmd(buf);
3211 },
3212 .struct_type => unreachable, // packed structs use `bitpack`
3213 else => unreachable,
3214 },2782 },
3215 .un => unreachable, // packed unions use `bitpack`2783 else => return cg.fail("TODO: Support intShr for integer bitsize: {d}", .{ty.bits}),
3216 .bitpack => |bitpack| return cg.lowerConstant(.fromInterned(bitpack.backing_int_val)),
3217 .memoized_call => unreachable,
3218 }2784 }
3219}2785}
32202786
3221/// Stores the value as a 128bit-immediate value by storing it inside2787fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3222/// the list and returning the index into this list as `WValue`.2788 if (!ty.is_signed) return operand;
3223fn storeSimdImmd(cg: *CodeGen, value: [16]u8) !WValue {2789 switch (ty.bits) {
3224 const index = @as(u32, @intCast(cg.simd_immediates.items.len));2790 0 => unreachable,
3225 try cg.simd_immediates.append(cg.gpa, value);2791 1...32 => {
3226 return .{ .imm128 = index };2792 try cg.emitWValue(operand);
3227}2793 try cg.addImm32(31);
2794 try cg.addTag(.i32_shr_s);
32282795
3229fn emitUndefined(cg: *CodeGen, ty: Type) InnerError!WValue {2796 var mask = try cg.allocLocal(Type.i32);
3230 const zcu = cg.pt.zcu;2797 defer mask.free(cg);
3231 switch (ty.zigTypeTag(zcu)) {2798 try cg.addLocal(.local_tee, mask.local.value);
3232 .bool, .error_set => return .{ .imm32 = 0xaaaaaaaa },2799
3233 .int, .@"enum" => switch (ty.intInfo(zcu).bits) {2800 try cg.emitWValue(operand);
3234 0...32 => return .{ .imm32 = 0xaaaaaaaa },2801 try cg.addTag(.i32_xor);
3235 33...64 => return .{ .imm64 = 0xaaaaaaaaaaaaaaaa },2802 try cg.emitWValue(mask);
3236 else => unreachable,2803 try cg.addTag(.i32_sub);
3237 },2804 return .stack;
3238 .float => switch (ty.floatBits(cg.target)) {
3239 16 => return .{ .imm32 = 0xaaaaaaaa },
3240 32 => return .{ .float32 = @as(f32, @bitCast(@as(u32, 0xaaaaaaaa))) },
3241 64 => return .{ .float64 = @as(f64, @bitCast(@as(u64, 0xaaaaaaaaaaaaaaaa))) },
3242 else => unreachable,
3243 },
3244 .pointer => switch (cg.ptr_size) {
3245 .wasm32 => return .{ .imm32 = 0xaaaaaaaa },
3246 .wasm64 => return .{ .imm64 = 0xaaaaaaaaaaaaaaaa },
3247 },
3248 .optional => {
3249 const pl_ty = ty.optionalChild(zcu);
3250 if (ty.optionalReprIsPayload(zcu)) {
3251 return cg.emitUndefined(pl_ty);
3252 }
3253 return .{ .imm32 = 0xaaaaaaaa };
3254 },
3255 .error_union => {
3256 return .{ .imm32 = 0xaaaaaaaa };
3257 },
3258 .@"struct", .@"union" => {
3259 const backing_int_ty = ty.bitpackBackingInt(zcu);
3260 return cg.emitUndefined(backing_int_ty);
3261 },2805 },
3262 else => return cg.fail("Wasm TODO: emitUndefined for type: {t}\n", .{ty.zigTypeTag(zcu)}),2806 33...64 => {
3263 }2807 try cg.emitWValue(operand);
3264}2808 try cg.addImm64(63);
2809 try cg.addTag(.i64_shr_s);
32652810
3266fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {2811 var mask = try cg.allocLocal(Type.i64);
3267 const block = cg.air.unwrapBlock(inst);2812 defer mask.free(cg);
3268 try cg.lowerBlock(inst, block.ty, block.body);2813 try cg.addLocal(.local_tee, mask.local.value);
3269}
32702814
3271fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const Air.Inst.Index) InnerError!void {2815 try cg.emitWValue(operand);
3272 const zcu = cg.pt.zcu;2816 try cg.addTag(.i64_xor);
3273 // if wasm_block_ty is non-empty, we create a register to store the temporary value2817 try cg.emitWValue(mask);
3274 const block_result: WValue = if (block_ty.hasRuntimeBits(zcu))2818 try cg.addTag(.i64_sub);
3275 try cg.allocLocal(block_ty)2819 return .stack;
3276 else2820 },
3277 .none;2821 65...128 => {
2822 const u128_ty: IntType = .{ .is_signed = false, .bits = 128 };
32782823
3279 try cg.startBlock(.block, .empty);2824 const mask = try cg.allocStack(Type.u128);
3280 // Here we set the current block idx, so breaks know the depth to jump2825 try cg.emitWValue(mask);
3281 // to when breaking out.2826 try cg.emitWValue(mask);
3282 try cg.blocks.putNoClobber(cg.gpa, inst, .{
3283 .label = cg.block_depth,
3284 .value = block_result,
3285 });
32862827
3287 try cg.genBody(body);2828 _ = try cg.load(operand, Type.u64, 8);
3288 try cg.endBlock();2829 try cg.addImm64(63);
2830 try cg.addTag(.i64_shr_s);
32892831
3290 const liveness = cg.liveness.getBlock(inst);2832 var tmp = try cg.allocLocal(Type.u64);
3291 try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.deaths.len);2833 defer tmp.free(cg);
2834 try cg.addLocal(.local_tee, tmp.local.value);
2835 try cg.store(.stack, .stack, Type.u64, mask.offset() + 0);
2836 try cg.emitWValue(tmp);
2837 try cg.store(.stack, .stack, Type.u64, mask.offset() + 8);
32922838
3293 return cg.finishAir(inst, block_result, &.{});2839 const a = try cg.intXor(u128_ty, operand, mask);
2840 const b = try cg.intSub(u128_ty, a, mask);
2841 return b;
2842 },
2843 else => return cg.fail("TODO: Support intAbs for integer bitsize: {d}", .{ty.bits}),
2844 }
3294}2845}
32952846
3296/// appends a new wasm block to the code section and increases the `block_depth` by 12847fn intMax(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3297fn startBlock(cg: *CodeGen, block_tag: std.wasm.Opcode, block_type: std.wasm.BlockType) !void {2848 try cg.lowerToStack(lhs);
3298 cg.block_depth += 1;2849 try cg.lowerToStack(rhs);
3299 try cg.addInst(.{2850 _ = try cg.intCmp(ty, .gt, lhs, rhs);
3300 .tag = Mir.Inst.Tag.fromOpcode(block_tag),2851 try cg.addTag(.select);
3301 .data = .{ .block_type = block_type },2852 return .stack;
3302 });
3303}2853}
33042854
3305/// Ends the current wasm block and decreases the `block_depth` by 12855fn intMin(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3306fn endBlock(cg: *CodeGen) !void {2856 try cg.lowerToStack(lhs);
3307 try cg.addTag(.end);2857 try cg.lowerToStack(rhs);
3308 cg.block_depth -= 1;2858 _ = try cg.intCmp(ty, .lt, lhs, rhs);
2859 try cg.addTag(.select);
2860 return .stack;
3309}2861}
33102862
3311fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {2863fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3312 const block = cg.air.unwrapBlock(inst);2864 switch (ty.bits) {
33132865 0 => unreachable,
3314 // result type of loop is always 'noreturn', meaning we can always2866 1...32 => {
3315 // emit the wasm type 'block_empty'.2867 if (ty.is_signed and ty.bits < 32) {
3316 try cg.startBlock(.loop, .empty);2868 const mask: u32 = ~@as(u32, 0) >> @intCast(32 - ty.bits);
33172869 _ = try cg.intAnd(.u32, operand, .{ .imm32 = mask });
3318 try cg.loops.putNoClobber(cg.gpa, inst, cg.block_depth);2870 } else {
3319 defer assert(cg.loops.remove(inst));2871 try cg.emitWValue(operand);
33202872 }
3321 try cg.genBody(block.body);2873 try cg.addTag(.i32_clz);
3322 try cg.endBlock();2874 if (ty.bits < 32) {
2875 try cg.addImm32(32 - ty.bits);
2876 try cg.addTag(.i32_sub);
2877 }
2878 return .stack;
2879 },
2880 33...64 => {
2881 if (ty.is_signed and ty.bits < 64) {
2882 const mask: u64 = ~@as(u64, 0) >> @intCast(64 - ty.bits);
2883 _ = try cg.intAnd(.u64, operand, .{ .imm64 = mask });
2884 } else {
2885 try cg.emitWValue(operand);
2886 }
2887 try cg.addTag(.i64_clz);
2888 try cg.addTag(.i32_wrap_i64);
2889 if (ty.bits < 64) {
2890 try cg.addImm32(64 - ty.bits);
2891 try cg.addTag(.i32_sub);
2892 }
2893 return .stack;
2894 },
2895 65...128 => {
2896 var msb = try (try cg.load(operand, Type.u64, 8)).toLocal(cg, Type.u64);
2897 defer msb.free(cg);
33232898
3324 return cg.finishAir(inst, .none, &.{});2899 try cg.emitWValue(msb);
2900 try cg.addTag(.i64_clz);
2901 _ = try cg.load(operand, Type.u64, 0);
2902 try cg.addTag(.i64_clz);
2903 try cg.emitWValue(.{ .imm64 = 64 });
2904 try cg.addTag(.i64_add);
2905 _ = try cg.intCmp(.u64, .neq, msb, .{ .imm64 = 0 });
2906 try cg.addTag(.select);
2907 try cg.addTag(.i32_wrap_i64);
2908 return .stack;
2909 },
2910 else => return cg.fail("TODO: Support intClz for integer bitsize: {d}", .{ty.bits}),
2911 }
3325}2912}
33262913
3327fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {2914fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3328 const cond_br = cg.air.unwrapCondBr(inst);2915 switch (ty.bits) {
3329 const condition = try cg.resolveInst(cond_br.condition);2916 0 => unreachable,
3330 const then_body = cond_br.then_body;2917 1...32 => {
3331 const else_body = cond_br.else_body;2918 if (ty.bits < 32) {
3332 const liveness_condbr = cg.liveness.getCondBr(inst);2919 _ = try cg.intOr(.u32, operand, .{ .imm32 = @as(u32, 1) << @intCast(ty.bits) });
33332920 } else {
3334 // result type is always noreturn, so use `block_empty` as type.2921 try cg.emitWValue(operand);
3335 try cg.startBlock(.block, .empty);2922 }
3336 // emit the conditional value2923 try cg.addTag(.i32_ctz);
3337 try cg.emitWValue(condition);2924 return .stack;
2925 },
2926 33...64 => {
2927 if (ty.bits < 64) {
2928 _ = try cg.intOr(.u64, operand, .{ .imm64 = @as(u64, 1) << @intCast(ty.bits) });
2929 } else {
2930 try cg.emitWValue(operand);
2931 }
2932 try cg.addTag(.i64_ctz);
2933 try cg.addTag(.i32_wrap_i64);
2934 return .stack;
2935 },
2936 65...128 => {
2937 var lsb = try (try cg.load(operand, Type.u64, 0)).toLocal(cg, Type.u64);
2938 defer lsb.free(cg);
33382939
3339 // we inserted the block in front of the condition2940 try cg.emitWValue(lsb);
3340 // so now check if condition matches. If not, break outside this block2941 try cg.addTag(.i64_ctz);
3341 // and continue with the then codepath
3342 try cg.addLabel(.br_if, 0);
33432942
3344 try cg.branches.ensureUnusedCapacity(cg.gpa, 2);2943 _ = try cg.load(operand, Type.u64, 8);
3345 {2944 if (ty.bits < 128) {
3346 cg.branches.appendAssumeCapacity(.{});2945 try cg.addImm64(@as(u64, 1) << @intCast(ty.bits - 64));
3347 try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, @as(u32, @intCast(liveness_condbr.else_deaths.len)));2946 try cg.addTag(.i64_or);
3348 defer {2947 }
3349 var else_stack = cg.branches.pop().?;2948 try cg.addTag(.i64_ctz);
3350 else_stack.deinit(cg.gpa);2949 try cg.addImm64(64);
3351 }2950 try cg.addTag(.i64_add);
3352 try cg.genBody(else_body);2951 _ = try cg.intCmp(.u64, .neq, lsb, .{ .imm64 = 0 });
3353 try cg.endBlock();2952 try cg.addTag(.select);
2953 try cg.addTag(.i32_wrap_i64);
2954 return .stack;
2955 },
2956 else => return cg.fail("TODO: Support intCtz for integer bitsize: {d}", .{ty.bits}),
3354 }2957 }
2958}
33552959
3356 // Outer block that matches the condition2960fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3357 {2961 switch (ty.bits) {
3358 cg.branches.appendAssumeCapacity(.{});2962 0 => unreachable,
3359 try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, @as(u32, @intCast(liveness_condbr.then_deaths.len)));2963 1...32 => {
3360 defer {2964 try cg.emitWValue(operand);
3361 var then_stack = cg.branches.pop().?;2965 if (ty.is_signed and ty.bits < 32) {
3362 then_stack.deinit(cg.gpa);2966 try cg.addImm32(32 - ty.bits);
3363 }2967 try cg.addTag(.i32_shl);
3364 try cg.genBody(then_body);2968 }
3365 }2969 try cg.addTag(.i32_popcnt);
2970 return .stack;
2971 },
2972 33...64 => {
2973 try cg.emitWValue(operand);
2974 if (ty.is_signed and ty.bits < 64) {
2975 try cg.addImm64(64 - ty.bits);
2976 try cg.addTag(.i64_shl);
2977 }
2978 try cg.addTag(.i64_popcnt);
2979 try cg.addTag(.i32_wrap_i64);
2980 return .stack;
2981 },
2982 65...128 => {
2983 _ = try cg.load(operand, Type.u64, 0);
2984 try cg.addTag(.i64_popcnt);
2985 _ = try cg.load(operand, Type.u64, 8);
2986 if (ty.is_signed and ty.bits < 128) {
2987 try cg.addImm64(128 - ty.bits);
2988 try cg.addTag(.i64_shl);
2989 }
2990 try cg.addTag(.i64_popcnt);
33662991
3367 return cg.finishAir(inst, .none, &.{});2992 try cg.addTag(.i64_add);
2993 try cg.addTag(.i32_wrap_i64);
2994 return .stack;
2995 },
2996 else => return cg.fail("TODO: Support intPopCount for integer bitsize: {d}", .{ty.bits}),
2997 }
3368}2998}
33692999
3370fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) InnerError!void {3000fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3371 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;3001 switch (ty.bits) {
3002 0 => unreachable,
3003 1...32 => {
3004 const intrin_ret = try cg.callIntrinsic(
3005 .__bitreversesi2,
3006 &.{.u32_type},
3007 Type.u32,
3008 &.{operand},
3009 );
3010 if (ty.bits == 32) return intrin_ret;
3011 return cg.intShr(ty, intrin_ret, .{ .imm32 = 32 - ty.bits });
3012 },
3013 33...64 => {
3014 const intrin_ret = try cg.callIntrinsic(
3015 .__bitreversedi2,
3016 &.{.u64_type},
3017 Type.u64,
3018 &.{operand},
3019 );
3020 if (ty.bits == 64) return intrin_ret;
3021 return cg.intShr(ty, intrin_ret, .{ .imm32 = 64 - ty.bits });
3022 },
3023 65...128 => {
3024 const tmp = try cg.allocStack(Type.u128);
33723025
3373 const lhs = try cg.resolveInst(bin_op.lhs);3026 try cg.emitWValue(tmp);
3374 const rhs = try cg.resolveInst(bin_op.rhs);3027 const hi = try cg.load(operand, Type.u64, 8);
3375 const operand_ty = cg.typeOf(bin_op.lhs);3028 const hi_rev = try cg.callIntrinsic(
3376 const result = try cg.cmp(lhs, rhs, operand_ty, op);3029 .__bitreversedi2,
3377 return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });3030 &.{.u64_type},
3378}3031 Type.u64,
3032 &.{hi},
3033 );
3034 try cg.emitWValue(hi_rev);
3035 try cg.store(.stack, .stack, Type.u64, tmp.offset());
33793036
3380/// Compares two operands.3037 try cg.emitWValue(tmp);
3381/// Asserts rhs is not a stack value when the lhs isn't a stack value either3038 const lo = try cg.load(operand, Type.u64, 0);
3382/// NOTE: This leaves the result on top of the stack, rather than a new local.3039 const lo_rev = try cg.callIntrinsic(
3383fn cmp(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, op: std.math.CompareOperator) InnerError!WValue {3040 .__bitreversedi2,
3384 assert(!(lhs != .stack and rhs == .stack));3041 &.{.u64_type},
3385 const zcu = cg.pt.zcu;3042 Type.u64,
3386 if (ty.zigTypeTag(zcu) == .optional and !ty.optionalReprIsPayload(zcu)) {3043 &.{lo},
3387 const payload_ty = ty.optionalChild(zcu);3044 );
3388 if (payload_ty.hasRuntimeBits(zcu)) {3045 try cg.emitWValue(lo_rev);
3389 // When we hit this case, we must check the value of optionals3046 try cg.store(.stack, .stack, Type.u64, tmp.offset() + 8);
3390 // that are not pointers. This means first checking against non-null for3047
3391 // both lhs and rhs, as well as checking the payload are matching of lhs and rhs3048 if (ty.bits < 128) {
3392 return cg.cmpOptionals(lhs, rhs, ty, op);3049 const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
3393 }3050 return cg.intShr(shift_ty, tmp, .{ .imm32 = 128 - ty.bits });
3394 } else if (ty.isAnyFloat()) {3051 } else {
3395 return cg.cmpFloat(ty, lhs, rhs, op);3052 return tmp;
3396 } else if (isByRef(ty, zcu, cg.target)) {3053 }
3397 return cg.cmpBigInt(lhs, rhs, ty, op);3054 },
3055 else => return cg.fail("TODO: Support intBitReverse for integer bitsize: {d}", .{ty.bits}),
3398 }3056 }
3057}
33993058
3400 const signedness: std.builtin.Signedness = blk: {3059fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3401 // by default we tell the operand type is unsigned (i.e. bools and enum values)3060 switch (ty.bits) {
3402 if (ty.zigTypeTag(zcu) != .int) break :blk .unsigned;3061 0 => unreachable,
3062 1...32 => {
3063 const intrin_ret = try cg.callIntrinsic(
3064 .__bswapsi2,
3065 &.{.u32_type},
3066 Type.u32,
3067 &.{operand},
3068 );
3069 if (ty.bits == 32) return intrin_ret;
3070 return cg.intShr(ty, intrin_ret, .{ .imm32 = 32 - ty.bits });
3071 },
3072 33...64 => {
3073 const intrin_ret = try cg.callIntrinsic(
3074 .__bswapdi2,
3075 &.{.u64_type},
3076 Type.u64,
3077 &.{operand},
3078 );
3079 if (ty.bits == 64) return intrin_ret;
3080 return cg.intShr(ty, intrin_ret, .{ .imm32 = 64 - ty.bits });
3081 },
3082 65...128 => {
3083 const tmp = try cg.allocStack(Type.u128);
34033084
3404 // incase of an actual integer, we emit the correct signedness3085 const low = try cg.load(operand, Type.u64, 0);
3405 break :blk ty.intInfo(zcu).signedness;3086 const high = try cg.load(operand, Type.u64, 8);
3406 };
34073087
3408 // ensure that when we compare pointers, we emit3088 const swap_low = try cg.callIntrinsic(
3409 // the true pointer of a stack value, rather than the stack pointer.3089 .__bswapdi2,
3410 try cg.lowerToStack(lhs);3090 &.{.u64_type},
3411 try cg.lowerToStack(rhs);3091 Type.u64,
3092 &.{low},
3093 );
3094 const swap_high = try cg.callIntrinsic(
3095 .__bswapdi2,
3096 &.{.u64_type},
3097 Type.u64,
3098 &.{high},
3099 );
34123100
3413 const opcode: std.wasm.Opcode = buildOpcode(.{3101 try cg.store(tmp, swap_low, Type.u64, tmp.offset() + 8);
3414 .valtype1 = typeToValtype(ty, zcu, cg.target),3102 try cg.store(tmp, swap_high, Type.u64, tmp.offset());
3415 .op = switch (op) {
3416 .lt => .lt,
3417 .lte => .le,
3418 .eq => .eq,
3419 .neq => .ne,
3420 .gte => .ge,
3421 .gt => .gt,
3422 },
3423 .signedness = signedness,
3424 });
3425 try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode));
34263103
3427 return .stack;3104 if (ty.bits < 128) {
3105 const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
3106 return cg.intShr(shift_ty, tmp, .{ .imm32 = 128 - ty.bits });
3107 } else {
3108 return tmp;
3109 }
3110 },
3111 else => return cg.fail("TODO: Support intByteSwap for integer bitsize: {d}", .{ty.bits}),
3112 }
3428}3113}
34293114
3430/// Compares two floats.3115fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3431/// NOTE: Leaves the result of the comparison on top of the stack.3116 switch (ty.bits) {
3432fn cmpFloat(cg: *CodeGen, ty: Type, lhs: WValue, rhs: WValue, cmp_op: std.math.CompareOperator) InnerError!WValue {3117 0 => unreachable,
3433 const float_bits = ty.floatBits(cg.target);3118 1...31 => {
34343119 try cg.emitWValue(operand);
3435 const op: Op = switch (cmp_op) {3120 if (ty.is_signed) {
3436 .lt => .lt,3121 try cg.addImm32(32 - ty.bits);
3437 .lte => .le,3122 try cg.addTag(.i32_shl);
3438 .eq => .eq,3123 try cg.addImm32(32 - ty.bits);
3439 .neq => .ne,3124 try cg.addTag(.i32_shr_s);
3440 .gte => .ge,3125 } else {
3441 .gt => .gt,3126 try cg.addImm32(~@as(u32, 0) >> @intCast(32 - ty.bits));
3442 };3127 try cg.addTag(.i32_and);
34433128 }
3444 switch (float_bits) {
3445 16 => {
3446 _ = try cg.fpext(lhs, Type.f16, Type.f32);
3447 _ = try cg.fpext(rhs, Type.f16, Type.f32);
3448 const opcode = buildOpcode(.{ .op = op, .valtype1 = .f32 });
3449 try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode));
3450 return .stack;3129 return .stack;
3451 },3130 },
3452 32, 64 => {3131 32 => return operand,
3453 try cg.emitWValue(lhs);3132 33...63 => {
3454 try cg.emitWValue(rhs);3133 try cg.emitWValue(operand);
3455 const val_type: std.wasm.Valtype = if (float_bits == 32) .f32 else .f64;3134 if (ty.is_signed) {
3456 const opcode = buildOpcode(.{ .op = op, .valtype1 = val_type });3135 try cg.addImm64(64 - ty.bits);
3457 try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode));3136 try cg.addTag(.i64_shl);
3137 try cg.addImm64(64 - ty.bits);
3138 try cg.addTag(.i64_shr_s);
3139 } else {
3140 try cg.addImm64(~@as(u64, 0) >> @intCast(64 - ty.bits));
3141 try cg.addTag(.i64_and);
3142 }
3458 return .stack;3143 return .stack;
3459 },3144 },
3460 80, 128 => {3145 64 => return operand,
3461 const intrinsic = floatCmpIntrinsic(cmp_op, float_bits);3146 65...127 => {
3462 const result = try cg.callIntrinsic(intrinsic, &.{ ty.ip_index, ty.ip_index }, Type.bool, &.{ lhs, rhs });3147 const result = try cg.allocStack(Type.u128);
3463 return cg.cmp(result, .{ .imm32 = 0 }, Type.i32, cmp_op);3148
3149 try cg.emitWValue(result);
3150 _ = try cg.load(operand, Type.u64, 0);
3151 try cg.store(.stack, .stack, Type.u64, result.offset());
3152
3153 try cg.emitWValue(result);
3154 _ = try cg.load(operand, Type.u64, 8);
3155 if (ty.is_signed) {
3156 try cg.addImm64(128 - ty.bits);
3157 try cg.addTag(.i64_shl);
3158 try cg.addImm64(128 - ty.bits);
3159 try cg.addTag(.i64_shr_s);
3160 } else {
3161 try cg.addImm64(~@as(u64, 0) >> @intCast(128 - ty.bits));
3162 try cg.addTag(.i64_and);
3163 }
3164 try cg.store(.stack, .stack, Type.u64, result.offset() + 8);
3165
3166 return result;
3464 },3167 },
3465 else => unreachable,3168 128 => return operand,
3169 else => return cg.fail("TODO: Support intWrap for integer bitsize: {d}", .{ty.bits}),
3466 }3170 }
3467}3171}
34683172
3469fn airCmpVector(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {3173fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3470 _ = inst;3174 if (int_ty.bits <= 32) {
3471 return cg.fail("TODO implement airCmpVector for wasm", .{});3175 if (int_ty.is_signed) {
3472}3176 return .{ .imm32 = (~@as(u32, 0) >> @intCast(32 - int_ty.bits)) >> 1 };
3177 } else {
3178 return .{ .imm32 = ~@as(u32, 0) >> @intCast(32 - int_ty.bits) };
3179 }
3180 } else if (int_ty.bits <= 64) {
3181 if (int_ty.is_signed) {
3182 return .{ .imm64 = (~@as(u64, 0) >> @intCast(64 - int_ty.bits)) >> 1 };
3183 } else {
3184 return .{ .imm64 = ~@as(u64, 0) >> @intCast(64 - int_ty.bits) };
3185 }
3186 } else {
3187 const result = try cg.allocStack(Type.u128);
3188 try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, Type.u64, 0);
34733189
3474fn airCmpLtErrorsLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {3190 if (int_ty.is_signed) {
3475 const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;3191 try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(128 - int_ty.bits)) >> 1 }, Type.u64, 8);
3476 const operand = try cg.resolveInst(un_op);3192 } else {
34773193 try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(128 - int_ty.bits) }, Type.u64, 8);
3478 try cg.emitWValue(operand);3194 }
3479 const pt = cg.pt;3195 return result;
3480 const err_int_ty = try pt.errorIntType();3196 }
3481 try cg.addTag(.errors_len);3197}
3482 const result = try cg.cmp(.stack, .stack, err_int_ty, .lt);
34833198
3484 return cg.finishAir(inst, result, &.{un_op});3199fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3200 if (!int_ty.is_signed) {
3201 return cg.intZeroValue(int_ty);
3202 }
3203 if (int_ty.bits <= 32) {
3204 return .{ .imm32 = ~@as(u32, 0) << @intCast(int_ty.bits - 1) };
3205 } else if (int_ty.bits <= 64) {
3206 return .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 1) };
3207 } else {
3208 const result = try cg.allocStack(Type.u128);
3209 try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0);
3210 try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 65) }, Type.u64, 8);
3211 return result;
3212 }
3485}3213}
34863214
3487fn airBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {3215fn intAddSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3488 const br = cg.air.instructions.items(.data)[@intFromEnum(inst)].br;3216 const raw_val = try cg.intAdd(int_ty, lhs, rhs);
3489 const block = cg.blocks.get(br.block_inst).?;3217 var op_val = try cg.toLocalInt(try cg.intWrap(int_ty, raw_val), int_ty);
3218 defer op_val.free(cg);
34903219
3491 // if operand has codegen bits we should break with a value3220 const max_val = try cg.intMaxValue(int_ty);
3492 if (block.value != .none) {
3493 const operand = try cg.resolveInst(br.operand);
3494 try cg.lowerToStack(operand);
3495 try cg.addLocal(.local_set, block.value.local.value);
3496 }
34973221
3498 // We map every block to its block index.3222 if (int_ty.is_signed) {
3499 // We then determine how far we have to jump to it by subtracting it from current block depth3223 const zero = try cg.intZeroValue(int_ty);
3500 const idx: u32 = cg.block_depth - block.label;3224 var rhs_is_neg = try cg.toLocalInt(try cg.intCmp(int_ty, .lt, rhs, zero), .u32);
3501 try cg.addLabel(.br, idx);3225 defer rhs_is_neg.free(cg);
3226 const min_val = try cg.intMinValue(int_ty);
35023227
3503 return cg.finishAir(inst, .none, &.{br.operand});3228 try cg.emitWValue(min_val);
3229 try cg.emitWValue(max_val);
3230 try cg.emitWValue(rhs_is_neg);
3231 try cg.addTag(.select);
3232
3233 try cg.emitWValue(op_val);
3234 const overflow_cmp = try cg.intCmp(int_ty, .lt, op_val, lhs);
3235 const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
3236 try cg.emitWValue(is_overflow);
3237 try cg.addTag(.select);
3238 return .stack;
3239 } else {
3240 try cg.emitWValue(max_val);
3241 try cg.emitWValue(op_val);
3242
3243 const is_overflow = try cg.intCmp(int_ty, .lt, op_val, lhs);
3244 try cg.emitWValue(is_overflow);
3245 try cg.addTag(.select);
3246 return .stack;
3247 }
3504}3248}
35053249
3506fn airRepeat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {3250fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3507 const repeat = cg.air.instructions.items(.data)[@intFromEnum(inst)].repeat;3251 const raw_val = try cg.intSub(int_ty, lhs, rhs);
3508 const loop_label = cg.loops.get(repeat.loop_inst).?;3252 var op_val = try cg.toLocalInt(try cg.intWrap(int_ty, raw_val), int_ty);
3253 defer op_val.free(cg);
35093254
3510 const idx: u32 = cg.block_depth - loop_label;3255 if (int_ty.is_signed) {
3511 try cg.addLabel(.br, idx);3256 const zero = try cg.intZeroValue(int_ty);
3257 var rhs_is_neg = try cg.toLocalInt(try cg.intCmp(int_ty, .lt, rhs, zero), .u32);
3258 defer rhs_is_neg.free(cg);
3259 const max_val = try cg.intMaxValue(int_ty);
3260 const min_val = try cg.intMinValue(int_ty);
35123261
3513 return cg.finishAir(inst, .none, &.{});3262 try cg.emitWValue(max_val);
3263 try cg.emitWValue(min_val);
3264 try cg.emitWValue(rhs_is_neg);
3265 try cg.addTag(.select);
3266
3267 try cg.emitWValue(op_val);
3268 const overflow_cmp = try cg.intCmp(int_ty, .gt, op_val, lhs);
3269 const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
3270 try cg.emitWValue(is_overflow);
3271 try cg.addTag(.select);
3272 return .stack;
3273 } else {
3274 const zero = try cg.intZeroValue(int_ty);
3275
3276 try cg.emitWValue(zero);
3277 try cg.emitWValue(op_val);
3278 const is_overflow = try cg.intCmp(int_ty, .lt, lhs, rhs);
3279 try cg.emitWValue(is_overflow);
3280 try cg.addTag(.select);
3281 return .stack;
3282 }
3514}3283}
35153284
3516fn airNot(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {3285fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3517 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3286 // Remove when > 128 int ops will be implemented in backend
3287 if (int_ty.bits == 128) {
3288 if (!int_ty.is_signed) {
3289 return cg.fail("TODO: mul_sat for unsigned 128-bit integers", .{});
3290 }
35183291
3519 const operand = try cg.resolveInst(ty_op.operand);3292 const overflow_ret = try cg.allocStack(Type.i32);
3520 const operand_ty = cg.typeOf(ty_op.operand);3293 const ret = try cg.callIntrinsic(
3521 const pt = cg.pt;3294 .__muloti4,
3522 const zcu = pt.zcu;3295 &[_]InternPool.Index{ .i128_type, .i128_type, .usize_type },
3296 Type.i128,
3297 &.{ lhs, rhs, overflow_ret },
3298 );
3299 try cg.lowerToStack(ret);
35233300
3524 const result = result: {3301 const xor = try cg.intXor(int_ty, lhs, rhs);
3525 if (operand_ty.zigTypeTag(zcu) == .bool) {3302 const sign_v = try cg.intShr(int_ty, xor, .{ .imm32 = 127 });
3526 try cg.emitWValue(operand);
3527 try cg.addTag(.i32_eqz);
3528 const not_tmp = try cg.allocLocal(operand_ty);
3529 try cg.addLocal(.local_set, not_tmp.local.value);
3530 break :result not_tmp;
3531 } else {
3532 const int_info = operand_ty.intInfo(zcu);
3533 const wasm_bits = toWasmBits(int_info.bits) orelse {
3534 return cg.fail("TODO: Implement binary NOT for {f}", .{operand_ty.fmt(pt)});
3535 };
35363303
3537 switch (wasm_bits) {3304 // xor ~@as(u127, 0)
3538 32 => {3305 try cg.emitWValue(sign_v);
3539 try cg.emitWValue(operand);3306 const lsb = try cg.load(sign_v, Type.u64, 0);
3540 try cg.addImm32(switch (int_info.signedness) {3307 _ = try cg.intXor(.u64, lsb, .{ .imm64 = ~@as(u64, 0) });
3541 .unsigned => ~@as(u32, 0) >> @intCast(32 - int_info.bits),3308 try cg.store(.stack, .stack, Type.u64, sign_v.offset());
3542 .signed => ~@as(u32, 0),
3543 });
3544 try cg.addTag(.i32_xor);
3545 break :result .stack;
3546 },
3547 64 => {
3548 try cg.emitWValue(operand);
3549 try cg.addImm64(switch (int_info.signedness) {
3550 .unsigned => ~@as(u64, 0) >> @intCast(64 - int_info.bits),
3551 .signed => ~@as(u64, 0),
3552 });
3553 try cg.addTag(.i64_xor);
3554 break :result .stack;
3555 },
3556 128 => {
3557 const ptr = try cg.allocStack(operand_ty);
35583309
3559 try cg.emitWValue(ptr);3310 try cg.emitWValue(sign_v);
3560 _ = try cg.load(operand, Type.u64, 0);3311 const msb = try cg.load(sign_v, Type.u64, 8);
3561 try cg.addImm64(~@as(u64, 0));3312 _ = try cg.intXor(.u64, msb, .{ .imm64 = ~@as(u64, 0) >> 1 });
3562 try cg.addTag(.i64_xor);3313 try cg.store(.stack, .stack, Type.u64, sign_v.offset() + 8);
3563 try cg.store(.stack, .stack, Type.u64, ptr.offset());
3564
3565 try cg.emitWValue(ptr);
3566 _ = try cg.load(operand, Type.u64, 8);
3567 try cg.addImm64(switch (int_info.signedness) {
3568 .unsigned => ~@as(u64, 0) >> @intCast(128 - int_info.bits),
3569 .signed => ~@as(u64, 0),
3570 });
3571 try cg.addTag(.i64_xor);
3572 try cg.store(.stack, .stack, Type.u64, ptr.offset() + 8);
3573
3574 break :result ptr;
3575 },
3576 else => unreachable,
3577 }
3578 }
3579 };
3580 return cg.finishAir(inst, result, &.{ty_op.operand});
3581}
35823314
3583fn airTrap(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {3315 try cg.lowerToStack(sign_v);
3584 try cg.addTag(.@"unreachable");3316 _ = try cg.load(overflow_ret, Type.i32, 0);
3585 return cg.finishAir(inst, .none, &.{});3317 try cg.addTag(.i32_eqz);
3586}3318 try cg.addTag(.select);
35873319
3588fn airBreakpoint(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {3320 return .stack;
3589 // unsupported by wasm itfunc. Can be implemented once we support DWARF3321 }
3590 // for wasm
3591 try cg.addTag(.@"unreachable");
3592 return cg.finishAir(inst, .none, &.{});
3593}
35943322
3595fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {3323 const ext_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = int_ty.bits * 2 };
3596 try cg.addTag(.@"unreachable");
3597 return cg.finishAir(inst, .none, &.{});
3598}
35993324
3600fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {3325 const lhs_ext = try cg.intCast(ext_ty, int_ty, lhs);
3601 const zcu = cg.pt.zcu;3326 const rhs_ext = try cg.intCast(ext_ty, int_ty, rhs);
3602 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3603 const operand = try cg.resolveInst(ty_op.operand);
3604 const wanted_ty = cg.typeOfIndex(inst);
3605 const given_ty = cg.typeOf(ty_op.operand);
36063327
3607 const bit_size = given_ty.bitSize(zcu);3328 var mul_ext = try cg.toLocalInt(try cg.intMul(ext_ty, lhs_ext, rhs_ext), ext_ty);
3608 const needs_wrapping = (given_ty.isSignedInt(zcu) != wanted_ty.isSignedInt(zcu)) and3329 defer mul_ext.free(cg);
3609 bit_size != 32 and bit_size != 64 and bit_size != 128;
36103330
3611 const result = result: {3331 var op_val = try cg.toLocalInt(try cg.intTrunc(int_ty, ext_ty, mul_ext), int_ty);
3612 if (given_ty.isAnyFloat() or wanted_ty.isAnyFloat()) {3332 defer op_val.free(cg);
3613 break :result try cg.bitcast(wanted_ty, given_ty, operand);3333 const max_val = try cg.intMaxValue(int_ty);
3614 }
36153334
3616 if (isByRef(given_ty, zcu, cg.target) and !isByRef(wanted_ty, zcu, cg.target)) {3335 if (int_ty.is_signed) {
3617 const loaded_memory = try cg.load(operand, wanted_ty, 0);3336 const min_val = try cg.intMinValue(int_ty);
3618 if (needs_wrapping) {
3619 break :result try cg.wrapOperand(loaded_memory, wanted_ty);
3620 } else {
3621 break :result loaded_memory;
3622 }
3623 }
3624 if (!isByRef(given_ty, zcu, cg.target) and isByRef(wanted_ty, zcu, cg.target)) {
3625 const stack_memory = try cg.allocStack(wanted_ty);
3626 try cg.store(stack_memory, operand, given_ty, 0);
3627 if (needs_wrapping) {
3628 break :result try cg.wrapOperand(stack_memory, wanted_ty);
3629 } else {
3630 break :result stack_memory;
3631 }
3632 }
36333337
3634 if (needs_wrapping) {3338 try cg.emitWValue(min_val);
3635 break :result try cg.wrapOperand(operand, wanted_ty);
3636 }
36373339
3638 break :result switch (operand) {3340 try cg.emitWValue(max_val);
3639 // for stack offset, return a pointer to this offset.3341 try cg.emitWValue(op_val);
3640 .stack_offset => try cg.buildPointerOffset(operand, 0, .new),3342 const max_ext = try cg.intCast(ext_ty, int_ty, max_val);
3641 else => cg.reuseOperand(ty_op.operand, operand),3343 const ov_pos = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);
3642 };3344 try cg.emitWValue(ov_pos);
3643 };3345 try cg.addTag(.select);
3644 return cg.finishAir(inst, result, &.{ty_op.operand});
3645}
36463346
3647fn bitcast(cg: *CodeGen, wanted_ty: Type, given_ty: Type, operand: WValue) InnerError!WValue {3347 const min_ext = try cg.intCast(ext_ty, int_ty, min_val);
3648 const zcu = cg.pt.zcu;3348 const ov_neg = try cg.intCmp(ext_ty, .gt, min_ext, mul_ext);
3649 // if we bitcast a float to or from an integer we must use the 'reinterpret' instruction3349 try cg.emitWValue(ov_neg);
3650 if (!(wanted_ty.isAnyFloat() or given_ty.isAnyFloat())) return operand;3350 try cg.addTag(.select);
3651 if (wanted_ty.ip_index == .f16_type or given_ty.ip_index == .f16_type) return operand;3351 return .stack;
3652 if (wanted_ty.bitSize(zcu) > 64) return operand;3352 } else {
3653 assert((wanted_ty.isInt(zcu) and given_ty.isAnyFloat()) or (wanted_ty.isAnyFloat() and given_ty.isInt(zcu)));3353 try cg.emitWValue(max_val);
36543354 try cg.emitWValue(op_val);
3655 const opcode = buildOpcode(.{3355 const max_ext = try cg.intCast(ext_ty, int_ty, max_val);
3656 .op = .reinterpret,3356 const is_overflow = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);
3657 .valtype1 = typeToValtype(wanted_ty, zcu, cg.target),3357 try cg.emitWValue(is_overflow);
3658 .valtype2 = typeToValtype(given_ty, zcu, cg.target),3358 try cg.addTag(.select);
3659 });3359 return .stack;
3660 try cg.emitWValue(operand);3360 }
3661 try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode));
3662 return .stack;
3663}3361}
36643362
3665fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {3363fn intShlSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3666 const zcu = cg.pt.zcu;3364 const raw_val = try cg.intShl(int_ty, lhs, rhs);
3667 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;3365 var op_val = try cg.toLocalInt(try cg.intWrap(int_ty, raw_val), int_ty);
3668 const extra = cg.air.extraData(Air.StructField, ty_pl.payload);3366 defer op_val.free(cg);
36693367
3670 const struct_ptr = try cg.resolveInst(extra.data.struct_operand);3368 var check_val = try cg.toLocalInt(try cg.intShr(int_ty, op_val, rhs), int_ty);
3671 const struct_ptr_ty = cg.typeOf(extra.data.struct_operand);3369 defer check_val.free(cg);
3672 const struct_ty = struct_ptr_ty.childType(zcu);
3673 const result = try cg.structFieldPtr(inst, extra.data.struct_operand, struct_ptr, struct_ptr_ty, struct_ty, extra.data.field_index);
3674 return cg.finishAir(inst, result, &.{extra.data.struct_operand});
3675}
36763370
3677fn airStructFieldPtrIndex(cg: *CodeGen, inst: Air.Inst.Index, index: u32) InnerError!void {3371 const max_val = try cg.intMaxValue(int_ty);
3678 const zcu = cg.pt.zcu;
3679 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3680 const struct_ptr = try cg.resolveInst(ty_op.operand);
3681 const struct_ptr_ty = cg.typeOf(ty_op.operand);
3682 const struct_ty = struct_ptr_ty.childType(zcu);
36833372
3684 const result = try cg.structFieldPtr(inst, ty_op.operand, struct_ptr, struct_ptr_ty, struct_ty, index);3373 if (int_ty.is_signed) {
3685 return cg.finishAir(inst, result, &.{ty_op.operand});3374 const zero = try cg.intZeroValue(int_ty);
3686}3375 const min_val = try cg.intMinValue(int_ty);
36873376
3688fn structFieldPtr(3377 try cg.emitWValue(min_val);
3689 cg: *CodeGen,3378 try cg.emitWValue(max_val);
3690 inst: Air.Inst.Index,3379 const lhs_is_neg = try cg.intCmp(int_ty, .lt, lhs, zero);
3691 ref: Air.Inst.Ref,3380 try cg.emitWValue(lhs_is_neg);
3692 struct_ptr: WValue,3381 try cg.addTag(.select);
3693 struct_ptr_ty: Type,
3694 struct_ty: Type,
3695 index: u32,
3696) InnerError!WValue {
3697 const pt = cg.pt;
3698 const zcu = pt.zcu;
3699 const result_ty = cg.typeOfIndex(inst);
3700 const struct_ptr_ty_info = struct_ptr_ty.ptrInfo(zcu);
37013382
3702 const offset = switch (struct_ty.containerLayout(zcu)) {3383 try cg.emitWValue(op_val);
3703 .@"packed" => switch (struct_ty.zigTypeTag(zcu)) {3384 const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);
3704 .@"struct" => offset: {3385 try cg.emitWValue(is_overflow);
3705 if (result_ty.ptrInfo(zcu).packed_offset.host_size != 0) {3386 try cg.addTag(.select);
3706 break :offset @as(u32, 0);3387 return .stack;
3707 }3388 } else {
3708 const struct_type = zcu.typeToStruct(struct_ty).?;3389 try cg.emitWValue(max_val);
3709 break :offset @divExact(zcu.structPackedFieldBitOffset(struct_type, index) + struct_ptr_ty_info.packed_offset.bit_offset, 8);3390 try cg.emitWValue(op_val);
3710 },3391 const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);
3711 .@"union" => 0,3392 try cg.emitWValue(is_overflow);
3712 else => unreachable,3393 try cg.addTag(.select);
3394 return .stack;
3395 }
3396}
3397
3398fn intZeroValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3399 switch (int_ty.bits) {
3400 0 => unreachable,
3401 1...32 => return .{ .imm32 = 0 },
3402 33...64 => return .{ .imm64 = 0 },
3403 65...128 => {
3404 const result = try cg.allocStack(Type.u128);
3405 try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0);
3406 try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8);
3407 return result;
3713 },3408 },
3714 else => struct_ty.structFieldOffset(index, zcu),3409 else => return cg.fail("TODO: Implement intZeroValue for integer bitsize: {d}", .{int_ty.bits}),
3715 };
3716 // save a load and store when we can simply reuse the operand
3717 if (offset == 0) {
3718 return cg.reuseOperand(ref, struct_ptr);
3719 }3410 }
3720 switch (struct_ptr) {3411}
3721 .stack_offset => |stack_offset| {3412
3722 return .{ .stack_offset = .{ .value = stack_offset.value + @as(u32, @intCast(offset)), .references = 1 } };3413fn toLocalInt(cg: *CodeGen, value: WValue, int_ty: IntType) InnerError!WValue {
3414 switch (value) {
3415 .stack => {
3416 const ty: Type = switch (int_ty.bits) {
3417 0 => unreachable,
3418 1...32 => .u32,
3419 33...64 => .u64,
3420 65...128 => .u128,
3421 else => return cg.fail("TODO: Support toLocalInt for integer bitsize: {d}", .{int_ty.bits}),
3422 };
3423 const new_local = try cg.allocLocal(ty);
3424 try cg.addLocal(.local_set, new_local.local.value);
3425 return new_local;
3723 },3426 },
3724 else => return cg.buildPointerOffset(struct_ptr, offset, .new),3427 .local, .stack_offset => return value,
3428 else => unreachable,
3725 }3429 }
3726}3430}
37273431
3728fn airStructFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {3432const OverflowResult = struct {
3729 const pt = cg.pt;3433 result: WValue,
3730 const zcu = pt.zcu;3434 ov: WValue,
3731 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;3435};
3732 const struct_field = cg.air.extraData(Air.StructField, ty_pl.payload).data;
37333436
3734 const struct_ty = cg.typeOf(struct_field.struct_operand);3437fn intAddOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
3735 const operand = try cg.resolveInst(struct_field.struct_operand);3438 switch (int_ty.bits) {
3736 const field_index = struct_field.field_index;3439 0 => unreachable,
3737 const field_ty = struct_ty.fieldType(field_index, zcu);3440 1...128 => {
3738 if (!field_ty.hasRuntimeBits(zcu)) return cg.finishAir(inst, .none, &.{struct_field.struct_operand});3441 const raw_result = try cg.intAdd(int_ty, lhs, rhs);
3442 const op_result = try cg.intWrap(int_ty, raw_result);
3443 const op_tmp = try cg.toLocalInt(op_result, int_ty);
37393444
3740 const result: WValue = switch (struct_ty.containerLayout(zcu)) {3445 const overflow_bit = if (int_ty.is_signed) blk: {
3741 .@"packed" => switch (struct_ty.zigTypeTag(zcu)) {3446 const zero = try cg.intZeroValue(int_ty);
3742 .@"struct" => result: {3447 const rhs_is_neg = try cg.intCmp(int_ty, .lt, rhs, zero);
3743 const packed_struct = zcu.typeToPackedStruct(struct_ty).?;3448 const overflow_cmp = try cg.intCmp(int_ty, .lt, op_tmp, lhs);
3744 const offset = zcu.structPackedFieldBitOffset(packed_struct, field_index);3449 break :blk try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
3745 const backing_ty = Type.fromInterned(packed_struct.packed_backing_int_type);3450 } else try cg.intCmp(int_ty, .lt, op_tmp, lhs);
3746 const host_bits = backing_ty.intInfo(zcu).bits;
3747
3748 const const_wvalue: WValue = if (33 <= host_bits and host_bits <= 64)
3749 .{ .imm64 = offset }
3750 else
3751 .{ .imm32 = offset };
3752
3753 // for first field we don't require any shifting
3754 const shifted_value = if (offset == 0)
3755 operand
3756 else
3757 try cg.binOp(operand, const_wvalue, backing_ty, .shr);
3758
3759 if (field_ty.zigTypeTag(zcu) == .float) {
3760 const int_type = try pt.intType(.unsigned, @as(u16, @intCast(field_ty.bitSize(zcu))));
3761 const truncated = try cg.trunc(shifted_value, int_type, backing_ty);
3762 break :result try cg.bitcast(field_ty, int_type, truncated);
3763 } else if (field_ty.isPtrAtRuntime(zcu) and packed_struct.field_types.len == 1) {
3764 // In this case we do not have to perform any transformations,
3765 // we can simply reuse the operand.
3766 break :result cg.reuseOperand(struct_field.struct_operand, operand);
3767 } else if (field_ty.isPtrAtRuntime(zcu)) {
3768 const int_type = try pt.intType(.unsigned, @as(u16, @intCast(field_ty.bitSize(zcu))));
3769 break :result try cg.trunc(shifted_value, int_type, backing_ty);
3770 }
3771 break :result try cg.trunc(shifted_value, field_ty, backing_ty);
3772 },
3773 .@"union" => result: {
3774 if (isByRef(struct_ty, zcu, cg.target)) {
3775 if (!isByRef(field_ty, zcu, cg.target)) {
3776 break :result try cg.load(operand, field_ty, 0);
3777 } else {
3778 const new_stack_val = try cg.allocStack(field_ty);
3779 try cg.store(new_stack_val, operand, field_ty, 0);
3780 break :result new_stack_val;
3781 }
3782 }
37833451
3784 const union_int_type = try pt.intType(.unsigned, @as(u16, @intCast(struct_ty.bitSize(zcu))));3452 return .{ .result = op_tmp, .ov = overflow_bit };
3785 if (field_ty.zigTypeTag(zcu) == .float) {
3786 const int_type = try pt.intType(.unsigned, @as(u16, @intCast(field_ty.bitSize(zcu))));
3787 const truncated = try cg.trunc(operand, int_type, union_int_type);
3788 break :result try cg.bitcast(field_ty, int_type, truncated);
3789 } else if (field_ty.isPtrAtRuntime(zcu)) {
3790 const int_type = try pt.intType(.unsigned, @as(u16, @intCast(field_ty.bitSize(zcu))));
3791 break :result try cg.trunc(operand, int_type, union_int_type);
3792 }
3793 break :result try cg.trunc(operand, field_ty, union_int_type);
3794 },
3795 else => unreachable,
3796 },
3797 else => result: {
3798 const offset = std.math.cast(u32, struct_ty.structFieldOffset(field_index, zcu)) orelse {
3799 return cg.fail("Field type '{f}' too big to fit into stack frame", .{field_ty.fmt(pt)});
3800 };
3801 if (isByRef(field_ty, zcu, cg.target)) {
3802 switch (operand) {
3803 .stack_offset => |stack_offset| {
3804 break :result .{ .stack_offset = .{ .value = stack_offset.value + offset, .references = 1 } };
3805 },
3806 else => break :result try cg.buildPointerOffset(operand, offset, .new),
3807 }
3808 }
3809 break :result try cg.load(operand, field_ty, offset);
3810 },3453 },
3811 };3454 else => return cg.fail("TODO: Support intAddOverflow for integer bitsize: {d}", .{int_ty.bits}),
38123455 }
3813 return cg.finishAir(inst, result, &.{struct_field.struct_operand});
3814}3456}
38153457
3816fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) InnerError!void {3458fn intSubOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
3817 const pt = cg.pt;3459 switch (int_ty.bits) {
3818 const zcu = pt.zcu;3460 0 => unreachable,
3461 1...128 => {
3462 const raw_result = try cg.intSub(int_ty, lhs, rhs);
3463 const op_result = try cg.intWrap(int_ty, raw_result);
3464 const op_tmp = try cg.toLocalInt(op_result, int_ty);
38193465
3820 const switch_br = cg.air.unwrapSwitch(inst);3466 const overflow_bit = if (int_ty.is_signed) blk: {
3821 const target_ty = cg.typeOf(switch_br.operand);3467 const zero = try cg.intZeroValue(int_ty);
3468 const rhs_is_neg = try cg.intCmp(int_ty, .lt, rhs, zero);
3469 const overflow_cmp = try cg.intCmp(int_ty, .gt, op_tmp, lhs);
3470 break :blk try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
3471 } else try cg.intCmp(int_ty, .gt, op_tmp, lhs);
38223472
3823 assert(target_ty.hasRuntimeBits(zcu));3473 return .{ .result = op_tmp, .ov = overflow_bit };
3474 },
3475 else => return cg.fail("TODO: Support intSubOverflow for integer bitsize: {d}", .{int_ty.bits}),
3476 }
3477}
38243478
3825 // swap target value with placeholder local, for dispatching3479fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
3826 const target = if (is_dispatch_loop) target: {3480 var overflow_bit = try cg.allocLocal(Type.u32);
3827 const initial_target = try cg.resolveInst(switch_br.operand);3481 try cg.addImm32(0);
3828 const target: WValue = try cg.allocLocal(target_ty);3482 try cg.addLocal(.local_set, overflow_bit.local.value);
3829 try cg.lowerToStack(initial_target);
3830 try cg.addLocal(.local_set, target.local.value);
38313483
3832 try cg.startBlock(.loop, .empty); // dispatch loop start3484 const result_val = if (int_ty.bits <= 32) blk: {
3833 try cg.blocks.putNoClobber(cg.gpa, inst, .{3485 const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 64 };
3834 .label = cg.block_depth,3486 const lhs_upcast = try cg.intCast(new_ty, int_ty, lhs);
3835 .value = target,3487 const rhs_upcast = try cg.intCast(new_ty, int_ty, rhs);
3836 });3488 const mul_raw = try cg.intMul(new_ty, lhs_upcast, rhs_upcast);
3489 const bin_op = try cg.toLocalInt(mul_raw, new_ty);
38373490
3838 break :target target;3491 const res = try cg.intTrunc(int_ty, new_ty, bin_op);
3839 } else try cg.resolveInst(switch_br.operand);3492 const res_tmp = try cg.toLocalInt(res, int_ty);
38403493
3841 const liveness = try cg.liveness.getSwitchBr(cg.gpa, inst, switch_br.cases_len + 1);3494 const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp);
3842 defer cg.gpa.free(liveness.deaths);3495 _ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op);
3496 try cg.addLocal(.local_set, overflow_bit.local.value);
3497 break :blk res_tmp;
3498 } else if (int_ty.bits <= 64) blk: {
3499 const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 128 };
3500 const lhs_upcast = try cg.intCast(new_ty, int_ty, lhs);
3501 const rhs_upcast = try cg.intCast(new_ty, int_ty, rhs);
3502 const mul_raw = try cg.intMul(new_ty, lhs_upcast, rhs_upcast);
3503 const bin_op = try cg.toLocalInt(mul_raw, new_ty);
3504
3505 const res = try cg.intTrunc(int_ty, new_ty, bin_op);
3506 const res_tmp = try cg.toLocalInt(res, int_ty);
3507
3508 const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp);
3509 _ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op);
3510 try cg.addLocal(.local_set, overflow_bit.local.value);
3511 break :blk res_tmp;
3512 } else if (int_ty.bits == 128 and !int_ty.is_signed) blk: {
3513 var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);
3514 defer lhs_lsb.free(cg);
3515 var lhs_msb = try (try cg.load(lhs, Type.u64, 8)).toLocal(cg, Type.u64);
3516 defer lhs_msb.free(cg);
3517 var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);
3518 defer rhs_lsb.free(cg);
3519 var rhs_msb = try (try cg.load(rhs, Type.u64, 8)).toLocal(cg, Type.u64);
3520 defer rhs_msb.free(cg);
38433521
3844 const has_else_body = switch_br.else_body_len != 0;3522 const zero: WValue = .{ .imm64 = 0 };
3845 const branch_count = switch_br.cases_len + 1; // if else branch is missing, we trap when failing all conditions
3846 try cg.branches.ensureUnusedCapacity(cg.gpa, switch_br.cases_len + @intFromBool(has_else_body));
38473523
3848 if (switch_br.cases_len == 0) {3524 const cross_1 = try cg.callIntrinsic(
3849 assert(has_else_body);3525 .__multi3,
3526 &[_]InternPool.Index{.i64_type} ** 4,
3527 Type.i128,
3528 &.{ lhs_msb, zero, rhs_lsb, zero },
3529 );
3530 const cross_2 = try cg.callIntrinsic(
3531 .__multi3,
3532 &[_]InternPool.Index{.i64_type} ** 4,
3533 Type.i128,
3534 &.{ rhs_msb, zero, lhs_lsb, zero },
3535 );
3536 const mul_lsb = try cg.callIntrinsic(
3537 .__multi3,
3538 &[_]InternPool.Index{.i64_type} ** 4,
3539 Type.i128,
3540 &.{ rhs_lsb, zero, lhs_lsb, zero },
3541 );
38503542
3851 var it = switch_br.iterateCases();3543 const rhs_msb_not_zero = try cg.intCmp(.u64, .neq, rhs_msb, zero);
3852 const else_body = it.elseBody();3544 const lhs_msb_not_zero = try cg.intCmp(.u64, .neq, lhs_msb, zero);
3545 const both_msb_not_zero = try cg.intAnd(.u32, rhs_msb_not_zero, lhs_msb_not_zero);
38533546
3854 cg.branches.appendAssumeCapacity(.{});3547 const cross_1_msb = try cg.load(cross_1, .u64, 8);
3855 const else_deaths = liveness.deaths.len - 1;3548 const cross_1_msb_not_zero = try cg.intCmp(.u64, .neq, cross_1_msb, zero);
3856 try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.deaths[else_deaths].len);3549 const cond_1 = try cg.intOr(.u32, both_msb_not_zero, cross_1_msb_not_zero);
3857 defer {
3858 var else_branch = cg.branches.pop().?;
3859 else_branch.deinit(cg.gpa);
3860 }
3861 try cg.genBody(else_body);
38623550
3863 if (is_dispatch_loop) {3551 const cross_2_msb = try cg.load(cross_2, Type.u64, 8);
3864 try cg.endBlock(); // dispatch loop end3552 const cross_2_msb_not_zero = try cg.intCmp(.u64, .neq, cross_2_msb, zero);
3865 }3553 const cond_2 = try cg.intOr(.u32, cond_1, cross_2_msb_not_zero);
3866 return cg.finishAir(inst, .none, &.{});
3867 }
38683554
3869 var min: ?Value = null;3555 const cross_1_lsb = try cg.load(cross_1, Type.u64, 0);
3870 var max: ?Value = null;3556 const cross_2_lsb = try cg.load(cross_2, Type.u64, 0);
3871 var branching_size: u32 = 0; // single item +1, range +23557 const cross_add = try cg.intAdd(.u64, cross_1_lsb, cross_2_lsb);
38723558
3873 {3559 var mul_lsb_msb = try (try cg.load(mul_lsb, Type.u64, 8)).toLocal(cg, Type.u64);
3874 var cases_it = switch_br.iterateCases();3560 defer mul_lsb_msb.free(cg);
3875 while (cases_it.next()) |case| {3561 var all_add = try (try cg.intAdd(.u64, cross_add, mul_lsb_msb)).toLocal(cg, Type.u64);
3876 for (case.items) |item| {3562 defer all_add.free(cg);
3877 const val = Value.fromInterned(item.toInterned().?);3563 const add_overflow = try cg.intCmp(.u64, .lt, all_add, mul_lsb_msb);
3878 if (min == null or val.compareHetero(.lt, min.?, zcu)) min = val;
3879 if (max == null or val.compareHetero(.gt, max.?, zcu)) max = val;
3880 branching_size += 1;
3881 }
3882 for (case.ranges) |range| {
3883 const low = Value.fromInterned(range[0].toInterned().?);
3884 if (min == null or low.compareHetero(.lt, min.?, zcu)) min = low;
3885 const high = Value.fromInterned(range[1].toInterned().?);
3886 if (max == null or high.compareHetero(.gt, max.?, zcu)) max = high;
3887 branching_size += 2;
3888 }
3889 }
3890 }
38913564
3892 var min_space: Value.BigIntSpace = undefined;3565 _ = try cg.intOr(.u32, cond_2, add_overflow);
3893 const min_bigint = min.?.toBigInt(&min_space, zcu);3566 try cg.addLocal(.local_set, overflow_bit.local.value);
3894 var max_space: Value.BigIntSpace = undefined;
3895 const max_bigint = max.?.toBigInt(&max_space, zcu);
3896 const limbs = try cg.gpa.alloc(
3897 std.math.big.Limb,
3898 @max(min_bigint.limbs.len, max_bigint.limbs.len) + 1,
3899 );
3900 defer cg.gpa.free(limbs);
39013567
3902 const width_maybe: ?u32 = width: {3568 const tmp_result = try cg.allocStack(Type.u128);
3903 var width_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };3569 try cg.emitWValue(tmp_result);
3904 width_bigint.sub(max_bigint, min_bigint);3570 const mul_lsb_lsb = try cg.load(mul_lsb, Type.u64, 0);
3905 width_bigint.addScalar(width_bigint.toConst(), 1);3571 try cg.store(.stack, mul_lsb_lsb, Type.u64, tmp_result.offset());
3906 break :width width_bigint.toConst().toInt(u32) catch null;3572 try cg.store(tmp_result, all_add, Type.u64, 8);
3907 };3573 break :blk tmp_result;
3574 } else if (int_ty.bits == 128 and int_ty.is_signed) blk: {
3575 const overflow_ret = try cg.allocStack(Type.i32);
3576 const res = try cg.callIntrinsic(
3577 .__muloti4,
3578 &[_]InternPool.Index{ .i128_type, .i128_type, .usize_type },
3579 Type.i128,
3580 &.{ lhs, rhs, overflow_ret },
3581 );
3582 _ = try cg.load(overflow_ret, Type.i32, 0);
3583 try cg.addLocal(.local_set, overflow_bit.local.value);
3584 break :blk res;
3585 } else return cg.fail("TODO: intMulOverflow for bitsize {d}", .{int_ty.bits});
39083586
3909 try cg.startBlock(.block, .empty); // whole switch block start3587 return .{ .result = result_val, .ov = .{ .local = overflow_bit.local } };
3588}
39103589
3911 for (0..branch_count) |_| {3590fn intShlOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
3912 try cg.startBlock(.block, .empty);3591 switch (int_ty.bits) {
3592 0 => unreachable,
3593 1...128 => {
3594 const raw_shl = try cg.intShl(int_ty, lhs, rhs);
3595 const wrapped_shl = try cg.intWrap(int_ty, raw_shl);
3596 const shl_tmp = try cg.toLocalInt(wrapped_shl, int_ty);
3597
3598 const shr = try cg.intShr(int_ty, shl_tmp, rhs);
3599 const overflow_bit = try cg.intCmp(int_ty, .neq, shr, lhs);
3600
3601 return .{ .result = shl_tmp, .ov = overflow_bit };
3602 },
3603 else => return cg.fail("TODO: Support intShlOverflow for integer bitsize: {d}", .{int_ty.bits}),
3913 }3604 }
3605}
39143606
3915 // Heuristic on deciding when to use .br_table instead of .br_if jump table3607fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) InnerError!WValue {
3916 // 1. Differences between lowest and highest values should fit into u323608 const src_bits: u16 = switch (src_ty.bits) {
3917 // 2. .br_table should be applied for "dense" switch, we test it by checking .br_if jumps will need more instructions3609 0 => unreachable,
3918 // 3. Do not use .br_table for tiny switches3610 1...32 => 32,
3919 const use_br_table = cond: {3611 33...64 => 64,
3920 const width = width_maybe orelse break :cond false;3612 65...128 => 128,
3921 if (width > 2 * branching_size) break :cond false;3613 else => unreachable,
3922 if (width < 2 or branch_count < 2) break :cond false;
3923 break :cond true;
3924 };3614 };
39253615
3926 if (use_br_table) {3616 const dest_bits: u16 = switch (dest_ty.bits) {
3927 const width = width_maybe.?;3617 0 => unreachable,
3618 1...32 => 32,
3619 33...64 => 64,
3620 65...128 => 128,
3621 else => unreachable,
3622 };
39283623
3929 const br_value_original = try cg.binOp(target, try cg.resolveValue(min.?), target_ty, .sub);3624 if (src_bits == dest_bits) {
3930 _ = try cg.intcast(br_value_original, target_ty, Type.u32);3625 return operand;
3626 }
39313627
3932 const jump_table: Mir.JumpTable = .{ .length = width + 1 };3628 if (src_bits == 64 and dest_bits == 32) {
3933 const table_extra_index = try cg.addExtra(jump_table);3629 try cg.emitWValue(operand);
3934 try cg.addInst(.{ .tag = .br_table, .data = .{ .payload = table_extra_index } });3630 try cg.addTag(.i32_wrap_i64);
3631 return .stack;
3632 } else if (src_bits == 32 and dest_bits == 64) {
3633 try cg.emitWValue(operand);
3634 try cg.addTag(if (dest_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
3635 return .stack;
3636 } else if (dest_bits == 128) {
3637 const stack_ptr = try cg.allocStack(Type.u128);
3638 try cg.emitWValue(stack_ptr);
39353639
3936 const branch_list = try cg.mir_extra.addManyAsSlice(cg.gpa, width + 1);3640 const lhs = if (src_bits == 32) blk: {
3937 @memset(branch_list, branch_count - 1);3641 const sign_ty: IntType = .{ .is_signed = dest_ty.is_signed, .bits = 64 };
3642 break :blk try (try cg.intCast(sign_ty, src_ty, operand)).toLocal(cg, Type.u64);
3643 } else operand;
39383644
3939 var cases_it = switch_br.iterateCases();3645 try cg.store(.stack, lhs, Type.u64, stack_ptr.offset());
3940 while (cases_it.next()) |case| {
3941 for (case.items) |item| {
3942 const val = Value.fromInterned(item.toInterned().?);
3943 var val_space: Value.BigIntSpace = undefined;
3944 const val_bigint = val.toBigInt(&val_space, zcu);
3945 var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
3946 index_bigint.sub(val_bigint, min_bigint);
3947 branch_list[index_bigint.toConst().toInt(u32) catch unreachable] = case.idx;
3948 }
3949 for (case.ranges) |range| {
3950 var low_space: Value.BigIntSpace = undefined;
3951 const low_bigint = Value.fromInterned(range[0].toInterned().?).toBigInt(&low_space, zcu);
3952 var high_space: Value.BigIntSpace = undefined;
3953 const high_bigint = Value.fromInterned(range[1].toInterned().?).toBigInt(&high_space, zcu);
3954 var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
3955 index_bigint.sub(low_bigint, min_bigint);
3956 const start = index_bigint.toConst().toInt(u32) catch unreachable;
3957 index_bigint.sub(high_bigint, min_bigint);
3958 const end = (index_bigint.toConst().toInt(u32) catch unreachable) + 1;
3959 @memset(branch_list[start..end], case.idx);
3960 }
3961 }
3962 } else {
3963 var cases_it = switch_br.iterateCases();
3964 while (cases_it.next()) |case| {
3965 for (case.items) |ref| {
3966 const val = try cg.resolveInst(ref);
3967 _ = try cg.cmp(target, val, target_ty, .eq);
3968 try cg.addLabel(.br_if, case.idx); // item match found
3969 }
3970 for (case.ranges) |range| {
3971 const low = try cg.resolveInst(range[0]);
3972 const high = try cg.resolveInst(range[1]);
39733646
3974 const gte = try cg.cmp(target, low, target_ty, .gte);3647 if (dest_ty.is_signed) {
3975 const lte = try cg.cmp(target, high, target_ty, .lte);3648 try cg.emitWValue(stack_ptr);
3976 _ = try cg.binOp(gte, lte, Type.bool, .@"and");3649 const shr = try cg.intShr(IntType.i64, lhs, .{ .imm32 = 63 });
3977 try cg.addLabel(.br_if, case.idx); // range match found3650 try cg.store(.stack, shr, Type.u64, 8 + stack_ptr.offset());
3978 }3651 } else {
3652 try cg.store(stack_ptr, .{ .imm64 = 0 }, Type.u64, 8);
3979 }3653 }
3980 try cg.addLabel(.br, branch_count - 1);
3981 }
39823654
3983 var cases_it = switch_br.iterateCases();3655 if (src_bits == 32) {
3984 while (cases_it.next()) |case| {3656 var tmp_lhs = lhs;
3985 try cg.endBlock();3657 tmp_lhs.free(cg);
3986
3987 cg.branches.appendAssumeCapacity(.{});
3988 try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.deaths[case.idx].len);
3989 defer {
3990 var case_branch = cg.branches.pop().?;
3991 case_branch.deinit(cg.gpa);
3992 }3658 }
3993 try cg.genBody(case.body);
3994
3995 try cg.addLabel(.br, branch_count - case.idx - 1); // matching case found and executed => exit switch
3996 }
3997
3998 try cg.endBlock();
3999 if (has_else_body) {
4000 const else_body = cases_it.elseBody();
40013659
4002 cg.branches.appendAssumeCapacity(.{});3660 return stack_ptr;
4003 const else_deaths = liveness.deaths.len - 1;
4004 try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.deaths[else_deaths].len);
4005 defer {
4006 var else_branch = cg.branches.pop().?;
4007 else_branch.deinit(cg.gpa);
4008 }
4009 try cg.genBody(else_body);
4010 } else {3661 } else {
4011 try cg.addTag(.@"unreachable");3662 const load_ty = if (dest_bits == 32) Type.u32 else Type.u64;
3663 return cg.load(operand, load_ty, 0);
4012 }3664 }
3665}
40133666
4014 try cg.endBlock(); // whole switch block end3667fn intTrunc(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) InnerError!WValue {
3668 var result = try cg.intCast(dest_ty, src_ty, operand);
40153669
4016 if (is_dispatch_loop) {3670 const dest_wasm_bits: u16 = switch (dest_ty.bits) {
4017 try cg.endBlock(); // dispatch loop end3671 0 => unreachable,
3672 1...32 => 32,
3673 33...64 => 64,
3674 65...128 => 128,
3675 else => return cg.fail("TODO: Implement wasm integer truncation for integer bitsize: {d}", .{dest_ty.bits}),
3676 };
3677
3678 if (dest_wasm_bits != dest_ty.bits) {
3679 result = try cg.intWrap(dest_ty, result);
4018 }3680 }
40193681
4020 return cg.finishAir(inst, .none, &.{});3682 return result;
4021}3683}
40223684
4023fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {3685const FloatType = enum {
4024 const br = cg.air.instructions.items(.data)[@intFromEnum(inst)].br;3686 f16,
4025 const switch_loop = cg.blocks.get(br.block_inst).?;3687 f32,
40263688 f64,
4027 const operand = try cg.resolveInst(br.operand);3689 f80,
4028 try cg.lowerToStack(operand);3690 f128,
4029 try cg.addLocal(.local_set, switch_loop.value.local.value);
40303691
4031 const idx: u32 = cg.block_depth - switch_loop.label;3692 fn fromType(cg: *CodeGen, ty: Type) FloatType {
4032 try cg.addLabel(.br, idx);3693 assert(ty.isRuntimeFloat());
3694 return switch (ty.floatBits(cg.target)) {
3695 16 => .f16,
3696 32 => .f32,
3697 64 => .f64,
3698 80 => .f80,
3699 128 => .f128,
3700 else => unreachable,
3701 };
3702 }
3703};
40333704
4034 return cg.finishAir(inst, .none, &.{br.operand});3705fn floatAdd(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
3706 switch (ty) {
3707 .f16 => return cg.callIntrinsic(.__addhf3, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
3708 .f32 => {
3709 try cg.emitWValue(lhs);
3710 try cg.emitWValue(rhs);
3711 try cg.addTag(.f32_add);
3712 return .stack;
3713 },
3714 .f64 => {
3715 try cg.emitWValue(lhs);
3716 try cg.emitWValue(rhs);
3717 try cg.addTag(.f64_add);
3718 return .stack;
3719 },
3720 .f80 => return cg.callIntrinsic(.__addxf3, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
3721 .f128 => return cg.callIntrinsic(.__addtf3, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
3722 }
4035}3723}
40363724
4037fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind: enum { value, ptr }) InnerError!void {3725fn floatSub(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4038 const zcu = cg.pt.zcu;3726 switch (ty) {
4039 const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;3727 .f16 => return cg.callIntrinsic(.__subhf3, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
4040 const operand = try cg.resolveInst(un_op);3728 .f32 => {
4041 const err_union_ty = switch (op_kind) {3729 try cg.emitWValue(lhs);
4042 .value => cg.typeOf(un_op),3730 try cg.emitWValue(rhs);
4043 .ptr => cg.typeOf(un_op).childType(zcu),3731 try cg.addTag(.f32_sub);
4044 };3732 return .stack;
4045 const pl_ty = err_union_ty.errorUnionPayload(zcu);3733 },
40463734 .f64 => {
4047 const result: WValue = result: {3735 try cg.emitWValue(lhs);
4048 if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {3736 try cg.emitWValue(rhs);
4049 switch (opcode) {3737 try cg.addTag(.f64_sub);
4050 .i32_ne => break :result .{ .imm32 = 0 },3738 return .stack;
4051 .i32_eq => break :result .{ .imm32 = 1 },3739 },
4052 else => unreachable,3740 .f80 => return cg.callIntrinsic(.__subxf3, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
4053 }3741 .f128 => return cg.callIntrinsic(.__subtf3, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
4054 }3742 }
4055
4056 try cg.emitWValue(operand);
4057 if (op_kind == .ptr or pl_ty.hasRuntimeBits(zcu)) {
4058 try cg.addMemArg(.i32_load16_u, .{
4059 .offset = operand.offset() + @as(u32, @intCast(errUnionErrorOffset(pl_ty, zcu))),
4060 .alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),
4061 });
4062 }
4063
4064 // Compare the error value with '0'
4065 try cg.addImm32(0);
4066 try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode));
4067 break :result .stack;
4068 };
4069 return cg.finishAir(inst, result, &.{un_op});
4070}3743}
40713744
4072/// E!T -> T op_is_ptr == false3745fn floatMul(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4073/// *(E!T) -> *T op_is_prt == true3746 switch (ty) {
4074fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!void {3747 .f16 => return cg.callIntrinsic(.__mulhf3, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
4075 const zcu = cg.pt.zcu;3748 .f32 => {
4076 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3749 try cg.emitWValue(lhs);
40773750 try cg.emitWValue(rhs);
4078 const operand = try cg.resolveInst(ty_op.operand);3751 try cg.addTag(.f32_mul);
4079 const op_ty = cg.typeOf(ty_op.operand);3752 return .stack;
4080 const eu_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty;3753 },
4081 const payload_ty = eu_ty.errorUnionPayload(zcu);3754 .f64 => {
40823755 try cg.emitWValue(lhs);
4083 const result: WValue = result: {3756 try cg.emitWValue(rhs);
4084 if (!payload_ty.hasRuntimeBits(zcu)) {3757 try cg.addTag(.f64_mul);
4085 if (op_is_ptr) {3758 return .stack;
4086 break :result cg.reuseOperand(ty_op.operand, operand);3759 },
4087 } else {3760 .f80 => return cg.callIntrinsic(.__mulxf3, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
4088 break :result .none;3761 .f128 => return cg.callIntrinsic(.__multf3, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
4089 }3762 }
4090 }
4091
4092 const pl_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu));
4093 if (op_is_ptr or isByRef(payload_ty, zcu, cg.target)) {
4094 break :result try cg.buildPointerOffset(operand, pl_offset, .new);
4095 } else {
4096 assert(isByRef(eu_ty, zcu, cg.target));
4097 break :result try cg.load(operand, payload_ty, pl_offset);
4098 }
4099 };
4100 return cg.finishAir(inst, result, &.{ty_op.operand});
4101}3763}
41023764
4103/// E!T -> E op_is_ptr == false3765fn floatMulAdd(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue, addend: WValue) InnerError!WValue {
4104/// *(E!T) -> E op_is_ptr == true3766 const mul_result = try cg.floatMul(ty, lhs, rhs);
4105/// NOTE: op_is_ptr will not change return type3767 return cg.floatAdd(ty, mul_result, addend);
4106fn airUnwrapErrUnionError(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!void {
4107 const zcu = cg.pt.zcu;
4108 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4109
4110 const operand = try cg.resolveInst(ty_op.operand);
4111 const op_ty = cg.typeOf(ty_op.operand);
4112 const eu_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty;
4113 const payload_ty = eu_ty.errorUnionPayload(zcu);
4114
4115 const result: WValue = result: {
4116 if (eu_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
4117 break :result .{ .imm32 = 0 };
4118 }
4119
4120 const err_offset: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu));
4121 if (op_is_ptr or isByRef(eu_ty, zcu, cg.target)) {
4122 break :result try cg.load(operand, Type.anyerror, err_offset);
4123 } else {
4124 assert(!payload_ty.hasRuntimeBits(zcu));
4125 break :result cg.reuseOperand(ty_op.operand, operand);
4126 }
4127 };
4128 return cg.finishAir(inst, result, &.{ty_op.operand});
4129}3768}
41303769
4131fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {3770fn floatDiv(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4132 const zcu = cg.pt.zcu;3771 switch (ty) {
4133 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3772 .f16 => return cg.callIntrinsic(.__divhf3, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
41343773 .f32 => {
4135 const operand = try cg.resolveInst(ty_op.operand);3774 try cg.emitWValue(lhs);
4136 const err_ty = cg.typeOfIndex(inst);3775 try cg.emitWValue(rhs);
41373776 try cg.addTag(.f32_div);
4138 const pl_ty = cg.typeOf(ty_op.operand);3777 return .stack;
4139 const result = result: {3778 },
4140 if (!pl_ty.hasRuntimeBits(zcu)) {3779 .f64 => {
4141 break :result cg.reuseOperand(ty_op.operand, operand);3780 try cg.emitWValue(lhs);
4142 }3781 try cg.emitWValue(rhs);
41433782 try cg.addTag(.f64_div);
4144 const err_union = try cg.allocStack(err_ty);3783 return .stack;
4145 const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);3784 },
4146 try cg.store(payload_ptr, operand, pl_ty, 0);3785 .f80 => return cg.callIntrinsic(.__divxf3, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
3786 .f128 => return cg.callIntrinsic(.__divtf3, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
3787 }
3788}
41473789
4148 // ensure we also write '0' to the error part, so any present stack value gets overwritten by it.3790fn floatRem(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4149 try cg.emitWValue(err_union);3791 switch (ty) {
4150 try cg.addImm32(0);3792 .f16 => return cg.callIntrinsic(.__fmodh, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
4151 const err_val_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));3793 .f32 => return cg.callIntrinsic(.fmodf, &.{ .f32_type, .f32_type }, Type.f32, &.{ lhs, rhs }),
4152 try cg.addMemArg(.i32_store16, .{3794 .f64 => return cg.callIntrinsic(.fmod, &.{ .f64_type, .f64_type }, Type.f64, &.{ lhs, rhs }),
4153 .offset = err_union.offset() + err_val_offset,3795 .f80 => return cg.callIntrinsic(.__fmodx, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
4154 .alignment = 2,3796 .f128 => return cg.callIntrinsic(.fmodq, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
4155 });3797 }
4156 break :result err_union;
4157 };
4158 return cg.finishAir(inst, result, &.{ty_op.operand});
4159}3798}
41603799
4161fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {3800// div_trunc(a, b) = trunc(a / b)
4162 const zcu = cg.pt.zcu;3801fn floatDivTrunc(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4163 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3802 const div_result = try cg.floatDiv(ty, lhs, rhs);
3803 return cg.floatTrunc(ty, div_result);
3804}
41643805
4165 const operand = try cg.resolveInst(ty_op.operand);3806// div_floor(a, b) = floor(a / b)
4166 const err_ty = ty_op.ty.toType();3807fn floatDivFloor(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4167 const pl_ty = err_ty.errorUnionPayload(zcu);3808 const div_result = try cg.floatDiv(ty, lhs, rhs);
3809 return cg.floatFloor(ty, div_result);
3810}
41683811
4169 const result = result: {3812// mod(a, b) = fmod(fmod(a, b) + b, b)
4170 if (!pl_ty.hasRuntimeBits(zcu)) {3813fn floatMod(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4171 break :result cg.reuseOperand(ty_op.operand, operand);3814 const r = try cg.floatRem(ty, lhs, rhs);
4172 }3815 const s = try cg.floatAdd(ty, r, rhs);
3816 return cg.floatRem(ty, s, rhs);
3817}
41733818
4174 const err_union = try cg.allocStack(err_ty);3819// wasm fN_max NaN semantics differ with Zig
4175 // store error value3820fn floatMax(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4176 try cg.store(err_union, operand, Type.anyerror, @intCast(errUnionErrorOffset(pl_ty, zcu)));3821 switch (ty) {
3822 .f16 => return cg.callIntrinsic(.__fmaxh, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
3823 .f32 => return cg.callIntrinsic(.fmaxf, &.{ .f32_type, .f32_type }, Type.f32, &.{ lhs, rhs }),
3824 .f64 => return cg.callIntrinsic(.fmax, &.{ .f64_type, .f64_type }, Type.f64, &.{ lhs, rhs }),
3825 .f80 => return cg.callIntrinsic(.__fmaxx, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
3826 .f128 => return cg.callIntrinsic(.fmaxq, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
3827 }
3828}
41773829
4178 // write 'undefined' to the payload3830// wasm fN_min NaN semantics differ with Zig
4179 const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);3831fn floatMin(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4180 const len = @as(u32, @intCast(err_ty.errorUnionPayload(zcu).abiSize(zcu)));3832 switch (ty) {
4181 try cg.memset(Type.u8, payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaa });3833 .f16 => return cg.callIntrinsic(.__fminh, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
3834 .f32 => return cg.callIntrinsic(.fminf, &.{ .f32_type, .f32_type }, Type.f32, &.{ lhs, rhs }),
3835 .f64 => return cg.callIntrinsic(.fmin, &.{ .f64_type, .f64_type }, Type.f64, &.{ lhs, rhs }),
3836 .f80 => return cg.callIntrinsic(.__fminx, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
3837 .f128 => return cg.callIntrinsic(.fminq, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
3838 }
3839}
41823840
4183 break :result err_union;3841fn floatSqrt(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4184 };3842 switch (ty) {
4185 return cg.finishAir(inst, result, &.{ty_op.operand});3843 .f16 => return cg.callIntrinsic(.__sqrth, &.{.f16_type}, Type.f16, &.{arg}),
3844 .f32 => {
3845 try cg.emitWValue(arg);
3846 try cg.addTag(.f32_sqrt);
3847 return .stack;
3848 },
3849 .f64 => {
3850 try cg.emitWValue(arg);
3851 try cg.addTag(.f64_sqrt);
3852 return .stack;
3853 },
3854 .f80 => return cg.callIntrinsic(.__sqrtx, &.{.f80_type}, Type.f80, &.{arg}),
3855 .f128 => return cg.callIntrinsic(.sqrtq, &.{.f128_type}, Type.f128, &.{arg}),
3856 }
4186}3857}
41873858
4188fn airIntcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {3859fn floatSin(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4189 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3860 switch (ty) {
3861 .f16 => return cg.callIntrinsic(.__sinh, &.{.f16_type}, Type.f16, &.{arg}),
3862 .f32 => return cg.callIntrinsic(.sinf, &.{.f32_type}, Type.f32, &.{arg}),
3863 .f64 => return cg.callIntrinsic(.sin, &.{.f64_type}, Type.f64, &.{arg}),
3864 .f80 => return cg.callIntrinsic(.__sinx, &.{.f80_type}, Type.f80, &.{arg}),
3865 .f128 => return cg.callIntrinsic(.sinq, &.{.f128_type}, Type.f128, &.{arg}),
3866 }
3867}
41903868
4191 const ty = ty_op.ty.toType();3869fn floatCos(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4192 const operand = try cg.resolveInst(ty_op.operand);3870 switch (ty) {
4193 const operand_ty = cg.typeOf(ty_op.operand);3871 .f16 => return cg.callIntrinsic(.__cosh, &.{.f16_type}, Type.f16, &.{arg}),
4194 const zcu = cg.pt.zcu;3872 .f32 => return cg.callIntrinsic(.cosf, &.{.f32_type}, Type.f32, &.{arg}),
4195 if (ty.zigTypeTag(zcu) == .vector or operand_ty.zigTypeTag(zcu) == .vector) {3873 .f64 => return cg.callIntrinsic(.cos, &.{.f64_type}, Type.f64, &.{arg}),
4196 return cg.fail("todo Wasm intcast for vectors", .{});3874 .f80 => return cg.callIntrinsic(.__cosx, &.{.f80_type}, Type.f80, &.{arg}),
3875 .f128 => return cg.callIntrinsic(.cosq, &.{.f128_type}, Type.f128, &.{arg}),
4197 }3876 }
4198 if (ty.abiSize(zcu) > 16 or operand_ty.abiSize(zcu) > 16) {3877}
4199 return cg.fail("todo Wasm intcast for bitsize > 128", .{});3878
3879fn floatTan(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
3880 switch (ty) {
3881 .f16 => return cg.callIntrinsic(.__tanh, &.{.f16_type}, Type.f16, &.{arg}),
3882 .f32 => return cg.callIntrinsic(.tanf, &.{.f32_type}, Type.f32, &.{arg}),
3883 .f64 => return cg.callIntrinsic(.tan, &.{.f64_type}, Type.f64, &.{arg}),
3884 .f80 => return cg.callIntrinsic(.__tanx, &.{.f80_type}, Type.f80, &.{arg}),
3885 .f128 => return cg.callIntrinsic(.tanq, &.{.f128_type}, Type.f128, &.{arg}),
4200 }3886 }
3887}
42013888
4202 const op_bits = toWasmBits(@intCast(operand_ty.bitSize(zcu))).?;3889fn floatExp(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4203 const wanted_bits = toWasmBits(@intCast(ty.bitSize(zcu))).?;3890 switch (ty) {
4204 const result = if (op_bits == wanted_bits)3891 .f16 => return cg.callIntrinsic(.__exph, &.{.f16_type}, Type.f16, &.{arg}),
4205 cg.reuseOperand(ty_op.operand, operand)3892 .f32 => return cg.callIntrinsic(.expf, &.{.f32_type}, Type.f32, &.{arg}),
4206 else3893 .f64 => return cg.callIntrinsic(.exp, &.{.f64_type}, Type.f64, &.{arg}),
4207 try cg.intcast(operand, operand_ty, ty);3894 .f80 => return cg.callIntrinsic(.__expx, &.{.f80_type}, Type.f80, &.{arg}),
3895 .f128 => return cg.callIntrinsic(.expq, &.{.f128_type}, Type.f128, &.{arg}),
3896 }
3897}
42083898
4209 return cg.finishAir(inst, result, &.{ty_op.operand});3899fn floatExp2(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
3900 switch (ty) {
3901 .f16 => return cg.callIntrinsic(.__exp2h, &.{.f16_type}, Type.f16, &.{arg}),
3902 .f32 => return cg.callIntrinsic(.exp2f, &.{.f32_type}, Type.f32, &.{arg}),
3903 .f64 => return cg.callIntrinsic(.exp2, &.{.f64_type}, Type.f64, &.{arg}),
3904 .f80 => return cg.callIntrinsic(.__exp2x, &.{.f80_type}, Type.f80, &.{arg}),
3905 .f128 => return cg.callIntrinsic(.exp2q, &.{.f128_type}, Type.f128, &.{arg}),
3906 }
4210}3907}
42113908
4212/// Upcasts or downcasts an integer based on the given and wanted types,3909fn floatLog(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4213/// and stores the result in a new operand.3910 switch (ty) {
4214/// Asserts type's bitsize <= 1283911 .f16 => return cg.callIntrinsic(.__logh, &.{.f16_type}, Type.f16, &.{arg}),
4215/// NOTE: May leave the result on the top of the stack.3912 .f32 => return cg.callIntrinsic(.logf, &.{.f32_type}, Type.f32, &.{arg}),
4216fn intcast(cg: *CodeGen, operand: WValue, given: Type, wanted: Type) InnerError!WValue {3913 .f64 => return cg.callIntrinsic(.log, &.{.f64_type}, Type.f64, &.{arg}),
4217 const zcu = cg.pt.zcu;3914 .f80 => return cg.callIntrinsic(.__logx, &.{.f80_type}, Type.f80, &.{arg}),
4218 const given_bitsize = @as(u16, @intCast(given.bitSize(zcu)));3915 .f128 => return cg.callIntrinsic(.logq, &.{.f128_type}, Type.f128, &.{arg}),
4219 const wanted_bitsize = @as(u16, @intCast(wanted.bitSize(zcu)));
4220 assert(given_bitsize <= 128);
4221 assert(wanted_bitsize <= 128);
4222
4223 const op_bits = toWasmBits(given_bitsize).?;
4224 const wanted_bits = toWasmBits(wanted_bitsize).?;
4225 if (op_bits == wanted_bits) {
4226 return operand;
4227 }3916 }
3917}
42283918
4229 if (op_bits == 64 and wanted_bits == 32) {3919fn floatLog2(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4230 try cg.emitWValue(operand);3920 switch (ty) {
4231 try cg.addTag(.i32_wrap_i64);3921 .f16 => return cg.callIntrinsic(.__log2h, &.{.f16_type}, Type.f16, &.{arg}),
4232 return .stack;3922 .f32 => return cg.callIntrinsic(.log2f, &.{.f32_type}, Type.f32, &.{arg}),
4233 } else if (op_bits == 32 and wanted_bits == 64) {3923 .f64 => return cg.callIntrinsic(.log2, &.{.f64_type}, Type.f64, &.{arg}),
4234 try cg.emitWValue(operand);3924 .f80 => return cg.callIntrinsic(.__log2x, &.{.f80_type}, Type.f80, &.{arg}),
4235 try cg.addTag(if (wanted.isSignedInt(zcu)) .i64_extend_i32_s else .i64_extend_i32_u);3925 .f128 => return cg.callIntrinsic(.log2q, &.{.f128_type}, Type.f128, &.{arg}),
4236 return .stack;3926 }
4237 } else if (wanted_bits == 128) {3927}
4238 // for 128bit integers we store the integer in the virtual stack, rather than a local
4239 const stack_ptr = try cg.allocStack(wanted);
4240 try cg.emitWValue(stack_ptr);
42413928
4242 // for 32 bit integers, we first coerce the value into a 64 bit integer before storing it3929fn floatLog10(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4243 // meaning less store operations are required.3930 switch (ty) {
4244 const lhs = if (op_bits == 32) blk: {3931 .f16 => return cg.callIntrinsic(.__log10h, &.{.f16_type}, Type.f16, &.{arg}),
4245 const sign_ty = if (wanted.isSignedInt(zcu)) Type.i64 else Type.u64;3932 .f32 => return cg.callIntrinsic(.log10f, &.{.f32_type}, Type.f32, &.{arg}),
4246 break :blk try (try cg.intcast(operand, given, sign_ty)).toLocal(cg, sign_ty);3933 .f64 => return cg.callIntrinsic(.log10, &.{.f64_type}, Type.f64, &.{arg}),
4247 } else operand;3934 .f80 => return cg.callIntrinsic(.__log10x, &.{.f80_type}, Type.f80, &.{arg}),
3935 .f128 => return cg.callIntrinsic(.log10q, &.{.f128_type}, Type.f128, &.{arg}),
3936 }
3937}
42483938
4249 // store lsb first3939fn floatFloor(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4250 try cg.store(.stack, lhs, Type.u64, 0 + stack_ptr.offset());3940 switch (ty) {
3941 .f16 => return cg.callIntrinsic(.__floorh, &.{.f16_type}, Type.f16, &.{arg}),
3942 .f32 => {
3943 try cg.emitWValue(arg);
3944 try cg.addTag(.f32_floor);
3945 return .stack;
3946 },
3947 .f64 => {
3948 try cg.emitWValue(arg);
3949 try cg.addTag(.f64_floor);
3950 return .stack;
3951 },
3952 .f80 => return cg.callIntrinsic(.__floorx, &.{.f80_type}, Type.f80, &.{arg}),
3953 .f128 => return cg.callIntrinsic(.floorq, &.{.f128_type}, Type.f128, &.{arg}),
3954 }
3955}
42513956
4252 // For signed integers we shift lsb by 63 (64bit integer - 1 sign bit) and store remaining value3957fn floatCeil(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4253 if (wanted.isSignedInt(zcu)) {3958 switch (ty) {
4254 try cg.emitWValue(stack_ptr);3959 .f16 => return cg.callIntrinsic(.__ceilh, &.{.f16_type}, Type.f16, &.{arg}),
4255 const shr = try cg.binOp(lhs, .{ .imm64 = 63 }, Type.i64, .shr);3960 .f32 => {
4256 try cg.store(.stack, shr, Type.u64, 8 + stack_ptr.offset());3961 try cg.emitWValue(arg);
4257 } else {3962 try cg.addTag(.f32_ceil);
4258 // Ensure memory of msb is zero'd3963 return .stack;
4259 try cg.store(stack_ptr, .{ .imm64 = 0 }, Type.u64, 8);3964 },
4260 }3965 .f64 => {
4261 return stack_ptr;3966 try cg.emitWValue(arg);
4262 } else return cg.load(operand, wanted, 0);3967 try cg.addTag(.f64_ceil);
3968 return .stack;
3969 },
3970 .f80 => return cg.callIntrinsic(.__ceilx, &.{.f80_type}, Type.f80, &.{arg}),
3971 .f128 => return cg.callIntrinsic(.ceilq, &.{.f128_type}, Type.f128, &.{arg}),
3972 }
4263}3973}
42643974
4265fn airIsNull(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind: enum { value, ptr }) InnerError!void {3975fn floatRound(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4266 const zcu = cg.pt.zcu;3976 switch (ty) {
4267 const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;3977 .f16 => return cg.callIntrinsic(.__roundh, &.{.f16_type}, Type.f16, &.{arg}),
4268 const operand = try cg.resolveInst(un_op);3978 .f32 => {
3979 try cg.emitWValue(arg);
3980 try cg.addTag(.f32_nearest);
3981 return .stack;
3982 },
3983 .f64 => {
3984 try cg.emitWValue(arg);
3985 try cg.addTag(.f64_nearest);
3986 return .stack;
3987 },
3988 .f80 => return cg.callIntrinsic(.__roundx, &.{.f80_type}, Type.f80, &.{arg}),
3989 .f128 => return cg.callIntrinsic(.roundq, &.{.f128_type}, Type.f128, &.{arg}),
3990 }
3991}
42693992
4270 const op_ty = cg.typeOf(un_op);3993fn floatTrunc(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4271 const optional_ty = if (op_kind == .ptr) op_ty.childType(zcu) else op_ty;3994 switch (ty) {
4272 const result = try cg.isNull(operand, optional_ty, opcode);3995 .f16 => return cg.callIntrinsic(.__trunch, &.{.f16_type}, Type.f16, &.{arg}),
4273 return cg.finishAir(inst, result, &.{un_op});3996 .f32 => {
3997 try cg.emitWValue(arg);
3998 try cg.addTag(.f32_trunc);
3999 return .stack;
4000 },
4001 .f64 => {
4002 try cg.emitWValue(arg);
4003 try cg.addTag(.f64_trunc);
4004 return .stack;
4005 },
4006 .f80 => return cg.callIntrinsic(.__truncx, &.{.f80_type}, Type.f80, &.{arg}),
4007 .f128 => return cg.callIntrinsic(.truncq, &.{.f128_type}, Type.f128, &.{arg}),
4008 }
4274}4009}
42754010
4276/// For a given type and operand, checks if it's considered `null`.4011fn floatNeg(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4277/// NOTE: Leaves the result on the stack4012 switch (ty) {
4278fn isNull(cg: *CodeGen, operand: WValue, optional_ty: Type, opcode: std.wasm.Opcode) InnerError!WValue {4013 .f16 => {
4279 const pt = cg.pt;4014 try cg.emitWValue(arg);
4280 const zcu = pt.zcu;4015 try cg.addImm32(0x8000);
4281 try cg.emitWValue(operand);4016 try cg.addTag(.i32_xor);
4282 const payload_ty = optional_ty.optionalChild(zcu);4017 return .stack;
4283 if (!optional_ty.optionalReprIsPayload(zcu)) {4018 },
4284 // When payload is zero-bits, we can treat operand as a value, rather than4019 .f32 => {
4285 // a pointer to the stack value4020 try cg.emitWValue(arg);
4286 if (payload_ty.hasRuntimeBits(zcu)) {4021 try cg.addTag(.f32_neg);
4287 const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {4022 return .stack;
4288 return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)});4023 },
4289 };4024 .f64 => {
4290 try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = 1 });4025 try cg.emitWValue(arg);
4291 }4026 try cg.addTag(.f64_neg);
4292 } else if (payload_ty.isSlice(zcu)) {4027 return .stack;
4293 switch (cg.ptr_size) {4028 },
4294 .wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = 4 }),4029 .f80 => {
4295 .wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = 8 }),4030 const result = try cg.allocStack(Type.f80);
4296 }4031 try cg.emitWValue(result);
4032 try cg.emitWValue(arg);
4033 try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 });
4034 try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 });
4035 try cg.emitWValue(result);
4036 try cg.emitWValue(arg);
4037 try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 });
4038 try cg.addImm64(0x8000);
4039 try cg.addTag(.i64_xor);
4040 try cg.addMemArg(.i64_store16, .{ .offset = 8 + result.offset(), .alignment = 2 });
4041 return result;
4042 },
4043 .f128 => {
4044 const result = try cg.allocStack(Type.f128);
4045 try cg.emitWValue(result);
4046 try cg.emitWValue(arg);
4047 try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 });
4048 try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 });
4049 try cg.emitWValue(result);
4050 try cg.emitWValue(arg);
4051 try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 });
4052 try cg.addImm64(0x8000000000000000);
4053 try cg.addTag(.i64_xor);
4054 try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 });
4055 return result;
4056 },
4297 }4057 }
4298
4299 // Compare the null value with '0'
4300 try cg.addImm32(0);
4301 try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode));
4302
4303 return .stack;
4304}4058}
43054059
4306fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {4060fn floatAbs(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4307 const zcu = cg.pt.zcu;4061 switch (ty) {
4308 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4062 .f16 => return cg.callIntrinsic(.__fabsh, &.{.f16_type}, Type.f16, &.{arg}),
4309 const opt_ty = cg.typeOf(ty_op.operand);4063 .f32 => {
4310 const payload_ty = cg.typeOfIndex(inst);4064 try cg.emitWValue(arg);
4311 if (!payload_ty.hasRuntimeBits(zcu)) {4065 try cg.addTag(.f32_abs);
4312 return cg.finishAir(inst, .none, &.{ty_op.operand});4066 return .stack;
4067 },
4068 .f64 => {
4069 try cg.emitWValue(arg);
4070 try cg.addTag(.f64_abs);
4071 return .stack;
4072 },
4073 .f80 => return cg.callIntrinsic(.__fabsx, &.{.f80_type}, Type.f80, &.{arg}),
4074 .f128 => return cg.callIntrinsic(.fabsq, &.{.f128_type}, Type.f128, &.{arg}),
4313 }4075 }
4314
4315 const result = result: {
4316 const operand = try cg.resolveInst(ty_op.operand);
4317 if (opt_ty.optionalReprIsPayload(zcu)) break :result cg.reuseOperand(ty_op.operand, operand);
4318
4319 if (isByRef(payload_ty, zcu, cg.target)) {
4320 break :result try cg.buildPointerOffset(operand, 0, .new);
4321 }
4322
4323 break :result try cg.load(operand, payload_ty, 0);
4324 };
4325 return cg.finishAir(inst, result, &.{ty_op.operand});
4326}4076}
43274077
4328fn airOptionalPayloadPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {4078fn floatExtendCast(cg: *CodeGen, dest_ty: FloatType, src_ty: FloatType, operand: WValue) InnerError!WValue {
4329 const zcu = cg.pt.zcu;4079 switch (dest_ty) {
4330 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4080 .f16 => unreachable,
4331 const operand = try cg.resolveInst(ty_op.operand);4081 .f32 => switch (src_ty) {
4332 const opt_ty = cg.typeOf(ty_op.operand).childType(zcu);4082 .f16 => {
43334083 _ = try cg.callIntrinsic(.__extendhfsf2, &.{.f16_type}, Type.f32, &.{operand});
4334 const result = result: {4084 return .stack;
4335 const payload_ty = opt_ty.optionalChild(zcu);4085 },
4336 if (!payload_ty.hasRuntimeBits(zcu) or opt_ty.optionalReprIsPayload(zcu)) {4086 else => unreachable,
4337 break :result cg.reuseOperand(ty_op.operand, operand);4087 },
4338 }4088 .f64 => switch (src_ty) {
43394089 .f16 => {
4340 break :result try cg.buildPointerOffset(operand, 0, .new);4090 _ = try cg.callIntrinsic(.__extendhfsf2, &.{.f16_type}, Type.f32, &.{operand});
4341 };4091 try cg.addTag(.f64_promote_f32);
4342 return cg.finishAir(inst, result, &.{ty_op.operand});4092 return .stack;
4093 },
4094 .f32 => {
4095 try cg.emitWValue(operand);
4096 try cg.addTag(.f64_promote_f32);
4097 return .stack;
4098 },
4099 else => unreachable,
4100 },
4101 .f80 => switch (src_ty) {
4102 .f16 => return cg.callIntrinsic(.__extendhfxf2, &.{.f16_type}, Type.f80, &.{operand}),
4103 .f32 => return cg.callIntrinsic(.__extendsfxf2, &.{.f32_type}, Type.f80, &.{operand}),
4104 .f64 => return cg.callIntrinsic(.__extenddfxf2, &.{.f64_type}, Type.f80, &.{operand}),
4105 else => unreachable,
4106 },
4107 .f128 => switch (src_ty) {
4108 .f16 => return cg.callIntrinsic(.__extendhftf2, &.{.f16_type}, Type.f128, &.{operand}),
4109 .f32 => return cg.callIntrinsic(.__extendsftf2, &.{.f32_type}, Type.f128, &.{operand}),
4110 .f64 => return cg.callIntrinsic(.__extenddftf2, &.{.f64_type}, Type.f128, &.{operand}),
4111 .f80 => return cg.callIntrinsic(.__extendxftf2, &.{.f80_type}, Type.f128, &.{operand}),
4112 else => unreachable,
4113 },
4114 }
4343}4115}
43444116
4345fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {4117fn floatTruncCast(cg: *CodeGen, dest_ty: FloatType, src_ty: FloatType, operand: WValue) InnerError!WValue {
4346 const pt = cg.pt;4118 switch (dest_ty) {
4347 const zcu = pt.zcu;4119 .f16 => switch (src_ty) {
4348 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4120 .f32 => return cg.callIntrinsic(.__truncsfhf2, &.{.f32_type}, Type.f16, &.{operand}),
4349 const operand = try cg.resolveInst(ty_op.operand);4121 .f64 => {
4350 const opt_ty = cg.typeOf(ty_op.operand).childType(zcu);4122 try cg.emitWValue(operand);
4351 const payload_ty = opt_ty.optionalChild(zcu);4123 try cg.addTag(.f32_demote_f64);
43524124 return cg.callIntrinsic(.__truncsfhf2, &.{.f32_type}, Type.f16, &.{.stack});
4353 if (opt_ty.optionalReprIsPayload(zcu)) {4125 },
4354 return cg.finishAir(inst, operand, &.{ty_op.operand});4126 .f80 => return cg.callIntrinsic(.__truncxfhf2, &.{.f80_type}, Type.f16, &.{operand}),
4127 .f128 => return cg.callIntrinsic(.__trunctfhf2, &.{.f128_type}, Type.f16, &.{operand}),
4128 else => unreachable,
4129 },
4130 .f32 => switch (src_ty) {
4131 .f64 => {
4132 try cg.emitWValue(operand);
4133 try cg.addTag(.f32_demote_f64);
4134 return .stack;
4135 },
4136 .f80 => return cg.callIntrinsic(.__truncxfsf2, &.{.f80_type}, Type.f32, &.{operand}),
4137 .f128 => return cg.callIntrinsic(.__trunctfsf2, &.{.f128_type}, Type.f32, &.{operand}),
4138 else => unreachable,
4139 },
4140 .f64 => switch (src_ty) {
4141 .f80 => return cg.callIntrinsic(.__truncxfdf2, &.{.f80_type}, Type.f64, &.{operand}),
4142 .f128 => return cg.callIntrinsic(.__trunctfdf2, &.{.f128_type}, Type.f64, &.{operand}),
4143 else => unreachable,
4144 },
4145 .f80 => switch (src_ty) {
4146 .f128 => return cg.callIntrinsic(.__trunctfxf2, &.{.f128_type}, Type.f80, &.{operand}),
4147 else => unreachable,
4148 },
4149 .f128 => unreachable,
4355 }4150 }
4151}
43564152
4357 const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {4153fn intFromFloat(cg: *CodeGen, dest_ty: IntType, src_ty: FloatType, operand: WValue) InnerError!WValue {
4358 return cg.fail("Optional type {f} too big to fit into stack frame", .{opt_ty.fmt(pt)});4154 switch (dest_ty.bits) {
4359 };4155 0 => unreachable,
43604156 1...32 => switch (src_ty) {
4361 try cg.emitWValue(operand);4157 .f16 => {
4362 try cg.addImm32(1);4158 const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixhfsi else .__fixunshfsi;
4363 try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = 1 });4159 return cg.callIntrinsic(intrinsic, &.{.f16_type}, Type.u32, &.{operand});
4160 },
4161 .f32 => {
4162 try cg.emitWValue(operand);
4163 try cg.addTag(if (dest_ty.is_signed) .i32_trunc_f32_s else .i32_trunc_f32_u);
4164 return .stack;
4165 },
4166 .f64 => {
4167 try cg.emitWValue(operand);
4168 try cg.addTag(if (dest_ty.is_signed) .i32_trunc_f64_s else .i32_trunc_f64_u);
4169 return .stack;
4170 },
4171 .f80 => {
4172 const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixxfsi else .__fixunsxfsi;
4173 return cg.callIntrinsic(intrinsic, &.{.f80_type}, Type.u32, &.{operand});
4174 },
4175 .f128 => {
4176 const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixtfsi else .__fixunstfsi;
4177 return cg.callIntrinsic(intrinsic, &.{.f128_type}, Type.u32, &.{operand});
4178 },
4179 },
4180 33...64 => switch (src_ty) {
4181 .f16 => {
4182 const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixhfdi else .__fixunshfdi;
4183 return cg.callIntrinsic(intrinsic, &.{.f16_type}, Type.u64, &.{operand});
4184 },
4185 .f32 => {
4186 try cg.emitWValue(operand);
4187 try cg.addTag(if (dest_ty.is_signed) .i64_trunc_f32_s else .i64_trunc_f32_u);
4188 return .stack;
4189 },
4190 .f64 => {
4191 try cg.emitWValue(operand);
4192 try cg.addTag(if (dest_ty.is_signed) .i64_trunc_f64_s else .i64_trunc_f64_u);
4193 return .stack;
4194 },
4195 .f80 => {
4196 const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixxfdi else .__fixunsxfdi;
4197 return cg.callIntrinsic(intrinsic, &.{.f80_type}, Type.u64, &.{operand});
4198 },
4199 .f128 => {
4200 const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixtfdi else .__fixunstfdi;
4201 return cg.callIntrinsic(intrinsic, &.{.f128_type}, Type.u64, &.{operand});
4202 },
4203 },
4204 65...128 => switch (src_ty) {
4205 .f16 => {
4206 const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixhfti else .__fixunshfti;
4207 return cg.callIntrinsic(intrinsic, &.{.f16_type}, Type.u128, &.{operand});
4208 },
4209 .f32 => {
4210 const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixsfti else .__fixunssfti;
4211 return cg.callIntrinsic(intrinsic, &.{.f32_type}, Type.u128, &.{operand});
4212 },
4213 .f64 => {
4214 const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixdfti else .__fixunsdfti;
4215 return cg.callIntrinsic(intrinsic, &.{.f64_type}, Type.u128, &.{operand});
4216 },
4217 .f80 => {
4218 const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixxfti else .__fixunsxfti;
4219 return cg.callIntrinsic(intrinsic, &.{.f80_type}, Type.u128, &.{operand});
4220 },
4221 .f128 => {
4222 const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixtfti else .__fixunstfti;
4223 return cg.callIntrinsic(intrinsic, &.{.f128_type}, Type.u128, &.{operand});
4224 },
4225 },
4226 else => return cg.fail("TODO: Support intFromFloat for integer bitsize: {d}", .{dest_ty.bits}),
4227 }
4228}
43644229
4365 const result = try cg.buildPointerOffset(operand, 0, .new);4230fn floatFromInt(cg: *CodeGen, dest_ty: FloatType, src_ty: IntType, operand: WValue) InnerError!WValue {
4366 return cg.finishAir(inst, result, &.{ty_op.operand});4231 switch (dest_ty) {
4232 .f16 => switch (src_ty.bits) {
4233 0 => unreachable,
4234 1...32 => {
4235 const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatsihf else .__floatunsihf;
4236 return cg.callIntrinsic(intrinsic, &.{.i32_type}, Type.f16, &.{operand});
4237 },
4238 33...64 => {
4239 const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatdihf else .__floatundihf;
4240 return cg.callIntrinsic(intrinsic, &.{.i64_type}, Type.f16, &.{operand});
4241 },
4242 65...128 => {
4243 const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattihf else .__floatuntihf;
4244 return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f16, &.{operand});
4245 },
4246 else => return cg.fail("TODO: Support floatFromInt for {d}-bit int to 16-bit float", .{src_ty.bits}),
4247 },
4248 .f32 => switch (src_ty.bits) {
4249 0 => unreachable,
4250 1...32 => {
4251 try cg.emitWValue(operand);
4252 try cg.addTag(if (src_ty.is_signed) .f32_convert_i32_s else .f32_convert_i32_u);
4253 return .stack;
4254 },
4255 33...64 => {
4256 try cg.emitWValue(operand);
4257 try cg.addTag(if (src_ty.is_signed) .f32_convert_i64_s else .f32_convert_i64_u);
4258 return .stack;
4259 },
4260 65...128 => {
4261 const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattisf else .__floatuntisf;
4262 return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f32, &.{operand});
4263 },
4264 else => return cg.fail("TODO: Support floatFromInt for {d}-bit int to 32-bit float", .{src_ty.bits}),
4265 },
4266 .f64 => switch (src_ty.bits) {
4267 0 => unreachable,
4268 1...32 => {
4269 try cg.emitWValue(operand);
4270 try cg.addTag(if (src_ty.is_signed) .f64_convert_i32_s else .f64_convert_i32_u);
4271 return .stack;
4272 },
4273 33...64 => {
4274 try cg.emitWValue(operand);
4275 try cg.addTag(if (src_ty.is_signed) .f64_convert_i64_s else .f64_convert_i64_u);
4276 return .stack;
4277 },
4278 65...128 => {
4279 const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattidf else .__floatuntidf;
4280 return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f64, &.{operand});
4281 },
4282 else => return cg.fail("TODO: Support floatFromInt for {d}-bit int to 64-bit float", .{src_ty.bits}),
4283 },
4284 .f80 => switch (src_ty.bits) {
4285 0 => unreachable,
4286 1...32 => {
4287 const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatsixf else .__floatunsixf;
4288 return cg.callIntrinsic(intrinsic, &.{.i32_type}, Type.f80, &.{operand});
4289 },
4290 33...64 => {
4291 const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatdixf else .__floatundixf;
4292 return cg.callIntrinsic(intrinsic, &.{.i64_type}, Type.f80, &.{operand});
4293 },
4294 65...128 => {
4295 const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattixf else .__floatuntixf;
4296 return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f80, &.{operand});
4297 },
4298 else => return cg.fail("TODO: Support floatFromInt for {d}-bit int to 80-bit float", .{src_ty.bits}),
4299 },
4300 .f128 => switch (src_ty.bits) {
4301 0 => unreachable,
4302 1...32 => {
4303 const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatsitf else .__floatunsitf;
4304 return cg.callIntrinsic(intrinsic, &.{.i32_type}, Type.f128, &.{operand});
4305 },
4306 33...64 => {
4307 const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatditf else .__floatunditf;
4308 return cg.callIntrinsic(intrinsic, &.{.i64_type}, Type.f128, &.{operand});
4309 },
4310 65...128 => {
4311 const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattitf else .__floatuntitf;
4312 return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f128, &.{operand});
4313 },
4314 else => return cg.fail("TODO: Support floatFromInt for {d}-bit int to 128-bit float", .{src_ty.bits}),
4315 },
4316 }
4367}4317}
43684318
4369fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {4319fn lowerPtr(cg: *CodeGen, ptr_val: InternPool.Index, prev_offset: u64) InnerError!WValue {
4370 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4371 const payload_ty = cg.typeOf(ty_op.operand);
4372 const pt = cg.pt;4320 const pt = cg.pt;
4373 const zcu = pt.zcu;4321 const zcu = pt.zcu;
43744322 const ptr = zcu.intern_pool.indexToKey(ptr_val).ptr;
4375 const result = result: {4323 const offset: u64 = prev_offset + ptr.byte_offset;
4376 if (!payload_ty.hasRuntimeBits(zcu)) {4324 return switch (ptr.base_addr) {
4377 const non_null_bit = try cg.allocStack(Type.u1);4325 .nav => |nav| return .{ .nav_ref = .{ .nav_index = nav, .offset = @intCast(offset) } },
4378 try cg.emitWValue(non_null_bit);4326 .uav => |uav| return .{ .uav_ref = .{ .ip_index = uav.val, .offset = @intCast(offset), .orig_ptr_ty = uav.orig_ty } },
4379 try cg.addImm32(1);4327 .int => return cg.lowerConstant(try pt.intValue(.usize, offset)),
4380 try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = 1 });4328 .eu_payload => |eu_ptr| try cg.lowerPtr(
4381 break :result non_null_bit;4329 eu_ptr,
4382 }4330 offset + codegen.errUnionPayloadOffset(
43834331 Value.fromInterned(eu_ptr).typeOf(zcu).childType(zcu),
4384 const operand = try cg.resolveInst(ty_op.operand);4332 zcu,
4385 const op_ty = cg.typeOfIndex(inst);4333 ),
4386 if (op_ty.optionalReprIsPayload(zcu)) {4334 ),
4387 break :result cg.reuseOperand(ty_op.operand, operand);4335 .opt_payload => |opt_ptr| return cg.lowerPtr(opt_ptr, offset),
4388 }4336 .field => |field| {
4389 const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {4337 const base_ptr = Value.fromInterned(field.base);
4390 return cg.fail("Optional type {f} too big to fit into stack frame", .{op_ty.fmt(pt)});4338 const base_ty = base_ptr.typeOf(zcu).childType(zcu);
4391 };4339 const field_off: u64 = switch (base_ty.zigTypeTag(zcu)) {
43924340 .pointer => off: {
4393 // Create optional type, set the non-null bit, and store the operand inside the optional type4341 assert(base_ty.isSlice(zcu));
4394 const result_ptr = try cg.allocStack(op_ty);4342 break :off switch (field.index) {
4395 try cg.emitWValue(result_ptr);4343 Value.slice_ptr_index => 0,
4396 try cg.addImm32(1);4344 Value.slice_len_index => @divExact(cg.target.ptrBitWidth(), 8),
4397 try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = 1 });4345 else => unreachable,
43984346 };
4399 const payload_ptr = try cg.buildPointerOffset(result_ptr, 0, .new);4347 },
4400 try cg.store(payload_ptr, operand, payload_ty, 0);4348 .@"struct" => switch (base_ty.containerLayout(zcu)) {
4401 break :result result_ptr;4349 .auto => base_ty.structFieldOffset(@intCast(field.index), zcu),
4350 .@"extern", .@"packed" => unreachable,
4351 },
4352 .@"union" => switch (base_ty.containerLayout(zcu)) {
4353 .auto => base_ty.structFieldOffset(@intCast(field.index), zcu),
4354 .@"extern", .@"packed" => unreachable,
4355 },
4356 else => unreachable,
4357 };
4358 return cg.lowerPtr(field.base, offset + field_off);
4359 },
4360 .arr_elem, .comptime_field, .comptime_alloc => unreachable,
4402 };4361 };
4403
4404 return cg.finishAir(inst, result, &.{ty_op.operand});
4405}4362}
44064363
4407fn airSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {4364/// Asserts that `isByRef` returns `false` for `val.typeOf(zcu)`.
4408 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;4365fn lowerConstant(cg: *CodeGen, val: Value) InnerError!WValue {
4409 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;4366 const pt = cg.pt;
4367 const zcu = pt.zcu;
4368 const ty = val.typeOf(zcu);
4369 assert(!isByRef(ty, zcu, cg.target));
4370 const ip = &zcu.intern_pool;
4371 if (val.isUndef(zcu)) return cg.emitUndefined(ty);
44104372
4411 const lhs = try cg.resolveInst(bin_op.lhs);4373 switch (ip.indexToKey(val.ip_index)) {
4412 const rhs = try cg.resolveInst(bin_op.rhs);4374 .int_type,
4413 const slice_ty = cg.typeOfIndex(inst);4375 .ptr_type,
4376 .array_type,
4377 .vector_type,
4378 .opt_type,
4379 .anyframe_type,
4380 .error_union_type,
4381 .simple_type,
4382 .struct_type,
4383 .tuple_type,
4384 .union_type,
4385 .opaque_type,
4386 .enum_type,
4387 .func_type,
4388 .error_set_type,
4389 .inferred_error_set_type,
4390 => unreachable, // types, not values
44144391
4415 const slice = try cg.allocStack(slice_ty);4392 .undef => unreachable, // handled above
4416 try cg.store(slice, lhs, Type.usize, 0);4393 .simple_value => |simple_value| switch (simple_value) {
4417 try cg.store(slice, rhs, Type.usize, cg.ptrSize());4394 .void,
4395 .null,
4396 .@"unreachable",
4397 => unreachable, // non-runtime values
4398 .false, .true => return .{ .imm32 = switch (simple_value) {
4399 .false => 0,
4400 .true => 1,
4401 else => unreachable,
4402 } },
4403 },
4404 .variable,
4405 .@"extern",
4406 .func,
4407 .enum_literal,
4408 => unreachable, // non-runtime values
4409 .int => {
4410 const int_info = ty.intInfo(zcu);
4411 switch (int_info.signedness) {
4412 .signed => switch (int_info.bits) {
4413 0...32 => return .{ .imm32 = @bitCast(@as(i32, @intCast(val.toSignedInt(zcu)))) },
4414 33...64 => return .{ .imm64 = @bitCast(val.toSignedInt(zcu)) },
4415 else => unreachable,
4416 },
4417 .unsigned => switch (int_info.bits) {
4418 0...32 => return .{ .imm32 = @intCast(val.toUnsignedInt(zcu)) },
4419 33...64 => return .{ .imm64 = val.toUnsignedInt(zcu) },
4420 else => unreachable,
4421 },
4422 }
4423 },
4424 .err => |err| {
4425 const int = try pt.getErrorValue(err.name);
4426 return .{ .imm32 = int };
4427 },
4428 .error_union => |error_union| {
4429 const err_int_ty = try pt.errorIntType();
4430 const err_val: Value = switch (error_union.val) {
4431 .err_name => |err_name| .fromInterned(try pt.intern(.{ .err = .{
4432 .ty = ty.errorUnionSet(zcu).toIntern(),
4433 .name = err_name,
4434 } })),
4435 .payload => try pt.intValue(err_int_ty, 0),
4436 };
4437 const payload_type = ty.errorUnionPayload(zcu);
4438 if (!payload_type.hasRuntimeBits(zcu)) {
4439 // We use the error type directly as the type.
4440 return cg.lowerConstant(err_val);
4441 }
44184442
4419 return cg.finishAir(inst, slice, &.{ bin_op.lhs, bin_op.rhs });4443 return cg.fail("Wasm TODO: lowerConstant error union with non-zero-bit payload type", .{});
4444 },
4445 .enum_tag => |enum_tag| return cg.lowerConstant(.fromInterned(enum_tag.int)),
4446 .float => |float| switch (float.storage) {
4447 .f16 => |f16_val| return .{ .imm32 = @as(u16, @bitCast(f16_val)) },
4448 .f32 => |f32_val| return .{ .float32 = f32_val },
4449 .f64 => |f64_val| return .{ .float64 = f64_val },
4450 else => unreachable,
4451 },
4452 .slice => unreachable, // isByRef == true
4453 .ptr => return cg.lowerPtr(val.toIntern(), 0),
4454 .opt => if (ty.optionalReprIsPayload(zcu)) {
4455 if (val.optionalValue(zcu)) |payload| {
4456 return cg.lowerConstant(payload);
4457 } else {
4458 return .{ .imm32 = 0 };
4459 }
4460 } else {
4461 return .{ .imm32 = @intFromBool(!val.isNull(zcu)) };
4462 },
4463 .aggregate => switch (ip.indexToKey(ty.ip_index)) {
4464 .array_type => return cg.fail("Wasm TODO: LowerConstant for {f}", .{ty.fmt(pt)}),
4465 .vector_type => {
4466 assert(determineSimdStoreStrategy(ty, zcu, cg.target) == .direct);
4467 var buf: [16]u8 = undefined;
4468 val.writeToMemory(pt, &buf) catch unreachable;
4469 return cg.storeSimdImmd(buf);
4470 },
4471 .struct_type => unreachable, // packed structs use `bitpack`
4472 else => unreachable,
4473 },
4474 .un => unreachable, // packed unions use `bitpack`
4475 .bitpack => |bitpack| return cg.lowerConstant(.fromInterned(bitpack.backing_int_val)),
4476 .memoized_call => unreachable,
4477 }
4420}4478}
44214479
4422fn airSliceLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {4480/// Stores the value as a 128bit-immediate value by storing it inside
4423 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4481/// the list and returning the index into this list as `WValue`.
44244482fn storeSimdImmd(cg: *CodeGen, value: [16]u8) !WValue {
4425 const operand = try cg.resolveInst(ty_op.operand);4483 const index = @as(u32, @intCast(cg.simd_immediates.items.len));
4426 return cg.finishAir(inst, try cg.sliceLen(operand), &.{ty_op.operand});4484 try cg.simd_immediates.append(cg.gpa, value);
4485 return .{ .imm128 = index };
4427}4486}
44284487
4429fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {4488fn emitUndefined(cg: *CodeGen, ty: Type) InnerError!WValue {
4430 const zcu = cg.pt.zcu;4489 const zcu = cg.pt.zcu;
4431 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;4490 switch (ty.zigTypeTag(zcu)) {
44324491 .bool, .error_set => return .{ .imm32 = 0xaaaaaaaa },
4433 const slice_ty = cg.typeOf(bin_op.lhs);4492 .int, .@"enum" => switch (ty.intInfo(zcu).bits) {
4434 const slice = try cg.resolveInst(bin_op.lhs);4493 0...32 => return .{ .imm32 = 0xaaaaaaaa },
4435 const index = try cg.resolveInst(bin_op.rhs);4494 33...64 => return .{ .imm64 = 0xaaaaaaaaaaaaaaaa },
4436 const elem_ty = slice_ty.childType(zcu);4495 else => unreachable,
4437 const elem_size = elem_ty.abiSize(zcu);4496 },
44384497 .float => switch (ty.floatBits(cg.target)) {
4439 // load pointer onto stack4498 16 => return .{ .imm32 = 0xaaaaaaaa },
4440 _ = try cg.load(slice, Type.usize, 0);4499 32 => return .{ .float32 = @as(f32, @bitCast(@as(u32, 0xaaaaaaaa))) },
44414500 64 => return .{ .float64 = @as(f64, @bitCast(@as(u64, 0xaaaaaaaaaaaaaaaa))) },
4442 // calculate index into slice4501 else => unreachable,
4443 try cg.emitWValue(index);4502 },
4444 try cg.addImm32(@intCast(elem_size));4503 .pointer => switch (cg.ptr_size) {
4445 try cg.addTag(.i32_mul);4504 .wasm32 => return .{ .imm32 = 0xaaaaaaaa },
4446 try cg.addTag(.i32_add);4505 .wasm64 => return .{ .imm64 = 0xaaaaaaaaaaaaaaaa },
44474506 },
4448 const elem_result = if (isByRef(elem_ty, zcu, cg.target))4507 .optional => {
4449 .stack4508 const pl_ty = ty.optionalChild(zcu);
4450 else4509 if (ty.optionalReprIsPayload(zcu)) {
4451 try cg.load(.stack, elem_ty, 0);4510 return cg.emitUndefined(pl_ty);
4511 }
4512 return .{ .imm32 = 0xaaaaaaaa };
4513 },
4514 .error_union => {
4515 return .{ .imm32 = 0xaaaaaaaa };
4516 },
4517 .@"struct", .@"union" => {
4518 const backing_int_ty = ty.bitpackBackingInt(zcu);
4519 return cg.emitUndefined(backing_int_ty);
4520 },
4521 else => return cg.fail("Wasm TODO: emitUndefined for type: {t}\n", .{ty.zigTypeTag(zcu)}),
4522 }
4523}
44524524
4453 return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });4525fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4526 const block = cg.air.unwrapBlock(inst);
4527 try cg.lowerBlock(inst, block.ty, block.body);
4454}4528}
44554529
4456fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {4530fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const Air.Inst.Index) InnerError!void {
4457 const zcu = cg.pt.zcu;4531 const zcu = cg.pt.zcu;
4458 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;4532 // if wasm_block_ty is non-empty, we create a register to store the temporary value
4459 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;4533 const block_result: WValue = if (block_ty.hasRuntimeBits(zcu))
44604534 try cg.allocLocal(block_ty)
4461 const elem_ty = ty_pl.ty.toType().childType(zcu);4535 else
4462 const elem_size = elem_ty.abiSize(zcu);4536 .none;
4463
4464 const slice = try cg.resolveInst(bin_op.lhs);
4465 const index = try cg.resolveInst(bin_op.rhs);
4466
4467 _ = try cg.load(slice, Type.usize, 0);
4468
4469 // calculate index into slice
4470 try cg.emitWValue(index);
4471 try cg.addImm32(@intCast(elem_size));
4472 try cg.addTag(.i32_mul);
4473 try cg.addTag(.i32_add);
44744537
4475 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });4538 try cg.startBlock(.block, .empty);
4476}4539 // Here we set the current block idx, so breaks know the depth to jump
4540 // to when breaking out.
4541 try cg.blocks.putNoClobber(cg.gpa, inst, .{
4542 .label = cg.block_depth,
4543 .value = block_result,
4544 });
44774545
4478fn airSlicePtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {4546 try cg.genBody(body);
4479 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4547 try cg.endBlock();
4480 const operand = try cg.resolveInst(ty_op.operand);
4481 return cg.finishAir(inst, try cg.slicePtr(operand), &.{ty_op.operand});
4482}
44834548
4484fn slicePtr(cg: *CodeGen, operand: WValue) InnerError!WValue {4549 const liveness = cg.liveness.getBlock(inst);
4485 const ptr = try cg.load(operand, Type.usize, 0);4550 try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.deaths.len);
4486 return ptr.toLocal(cg, Type.usize);
4487}
44884551
4489fn sliceLen(cg: *CodeGen, operand: WValue) InnerError!WValue {4552 return cg.finishAir(inst, block_result, &.{});
4490 const len = try cg.load(operand, Type.usize, cg.ptrSize());
4491 return len.toLocal(cg, Type.usize);
4492}4553}
44934554
4494fn airTrunc(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {4555/// appends a new wasm block to the code section and increases the `block_depth` by 1
4495 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4556fn startBlock(cg: *CodeGen, block_tag: std.wasm.Opcode, block_type: std.wasm.BlockType) !void {
4557 cg.block_depth += 1;
4558 try cg.addInst(.{
4559 .tag = Mir.Inst.Tag.fromOpcode(block_tag),
4560 .data = .{ .block_type = block_type },
4561 });
4562}
44964563
4497 const operand = try cg.resolveInst(ty_op.operand);4564/// Ends the current wasm block and decreases the `block_depth` by 1
4498 const wanted_ty: Type = ty_op.ty.toType();4565fn endBlock(cg: *CodeGen) !void {
4499 const op_ty = cg.typeOf(ty_op.operand);4566 try cg.addTag(.end);
4500 const zcu = cg.pt.zcu;4567 cg.block_depth -= 1;
4568}
45014569
4502 if (wanted_ty.zigTypeTag(zcu) == .vector or op_ty.zigTypeTag(zcu) == .vector) {4570fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4503 return cg.fail("TODO: trunc for vectors", .{});4571 const block = cg.air.unwrapBlock(inst);
4504 }
45054572
4506 const result = if (op_ty.bitSize(zcu) == wanted_ty.bitSize(zcu))4573 // result type of loop is always 'noreturn', meaning we can always
4507 cg.reuseOperand(ty_op.operand, operand)4574 // emit the wasm type 'block_empty'.
4508 else4575 try cg.startBlock(.loop, .empty);
4509 try cg.trunc(operand, wanted_ty, op_ty);
45104576
4511 return cg.finishAir(inst, result, &.{ty_op.operand});4577 try cg.loops.putNoClobber(cg.gpa, inst, cg.block_depth);
4512}4578 defer assert(cg.loops.remove(inst));
45134579
4514/// Truncates a given operand to a given type, discarding any overflown bits.4580 try cg.genBody(block.body);
4515/// NOTE: Resulting value is left on the stack.4581 try cg.endBlock();
4516fn trunc(cg: *CodeGen, operand: WValue, wanted_ty: Type, given_ty: Type) InnerError!WValue {
4517 const zcu = cg.pt.zcu;
4518 const given_bits = @as(u16, @intCast(given_ty.bitSize(zcu)));
4519 if (toWasmBits(given_bits) == null) {
4520 return cg.fail("TODO: Implement wasm integer truncation for integer bitsize: {d}", .{given_bits});
4521 }
45224582
4523 var result = try cg.intcast(operand, given_ty, wanted_ty);4583 return cg.finishAir(inst, .none, &.{});
4524 const wanted_bits = @as(u16, @intCast(wanted_ty.bitSize(zcu)));
4525 const wasm_bits = toWasmBits(wanted_bits).?;
4526 if (wasm_bits != wanted_bits) {
4527 result = try cg.wrapOperand(result, wanted_ty);
4528 }
4529 return result;
4530}4584}
45314585
4532fn airArrayToSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {4586fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4533 const zcu = cg.pt.zcu;4587 const cond_br = cg.air.unwrapCondBr(inst);
4534 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4588 const condition = try cg.resolveInst(cond_br.condition);
4589 const then_body = cond_br.then_body;
4590 const else_body = cond_br.else_body;
4591 const liveness_condbr = cg.liveness.getCondBr(inst);
45354592
4536 const operand = try cg.resolveInst(ty_op.operand);4593 // result type is always noreturn, so use `block_empty` as type.
4537 const array_ty = cg.typeOf(ty_op.operand).childType(zcu);4594 try cg.startBlock(.block, .empty);
4538 const slice_ty = ty_op.ty.toType();4595 // emit the conditional value
4596 try cg.emitWValue(condition);
45394597
4540 // create a slice on the stack4598 // we inserted the block in front of the condition
4541 const slice_local = try cg.allocStack(slice_ty);4599 // so now check if condition matches. If not, break outside this block
4600 // and continue with the then codepath
4601 try cg.addLabel(.br_if, 0);
45424602
4543 // store the array ptr in the slice4603 try cg.branches.ensureUnusedCapacity(cg.gpa, 2);
4544 if (array_ty.hasRuntimeBits(zcu)) {4604 {
4545 try cg.store(slice_local, operand, Type.usize, 0);4605 cg.branches.appendAssumeCapacity(.{});
4606 try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, @as(u32, @intCast(liveness_condbr.else_deaths.len)));
4607 defer {
4608 var else_stack = cg.branches.pop().?;
4609 else_stack.deinit(cg.gpa);
4610 }
4611 try cg.genBody(else_body);
4612 try cg.endBlock();
4546 }4613 }
45474614
4548 // store the length of the array in the slice4615 // Outer block that matches the condition
4549 const array_len: u32 = @intCast(array_ty.arrayLen(zcu));4616 {
4550 try cg.store(slice_local, .{ .imm32 = array_len }, Type.usize, cg.ptrSize());4617 cg.branches.appendAssumeCapacity(.{});
4618 try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, @as(u32, @intCast(liveness_condbr.then_deaths.len)));
4619 defer {
4620 var then_stack = cg.branches.pop().?;
4621 then_stack.deinit(cg.gpa);
4622 }
4623 try cg.genBody(then_body);
4624 }
45514625
4552 return cg.finishAir(inst, slice_local, &.{ty_op.operand});4626 return cg.finishAir(inst, .none, &.{});
4553}4627}
45544628
4555fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {4629fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) InnerError!void {
4556 const zcu = cg.pt.zcu;
4557 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;4630 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4631 const lhs = try cg.resolveInst(bin_op.lhs);
4632 const rhs = try cg.resolveInst(bin_op.rhs);
4633 const operand_ty = cg.typeOf(bin_op.lhs);
4634 const zcu = cg.pt.zcu;
45584635
4559 const ptr_ty = cg.typeOf(bin_op.lhs);4636 const type_tag = operand_ty.zigTypeTag(zcu);
4560 const ptr = try cg.resolveInst(bin_op.lhs);
4561 const index = try cg.resolveInst(bin_op.rhs);
4562 const elem_ty = ptr_ty.childType(zcu);
4563 const elem_size = elem_ty.abiSize(zcu);
45644637
4565 // load pointer onto the stack4638 if (type_tag == .vector) {
4566 if (ptr_ty.isSlice(zcu)) {4639 return cg.fail("TODO: implement AIR op: cmp for vectors", .{});
4567 _ = try cg.load(ptr, Type.usize, 0);
4568 } else {
4569 try cg.lowerToStack(ptr);
4570 }4640 }
45714641
4572 // calculate index into slice4642 if (type_tag == .optional and !operand_ty.optionalReprIsPayload(zcu)) {
4573 try cg.emitWValue(index);4643 const payload_ty = operand_ty.optionalChild(zcu);
4574 try cg.addImm32(@intCast(elem_size));
4575 try cg.addTag(.i32_mul);
4576 try cg.addTag(.i32_add);
4577
4578 const elem_result = if (isByRef(elem_ty, zcu, cg.target))
4579 .stack
4580 else
4581 try cg.load(.stack, elem_ty, 0);
45824644
4583 return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });4645 if (payload_ty.hasRuntimeBits(zcu)) {
4584}4646 assert(op == .eq or op == .neq);
4647 assert(!isByRef(payload_ty, zcu, cg.target));
45854648
4586fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {4649 var result = try cg.allocLocal(Type.i32);
4587 const zcu = cg.pt.zcu;4650 defer result.free(cg);
4588 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4589 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
45904651
4591 const ptr_ty = cg.typeOf(bin_op.lhs);4652 var lhs_null = try cg.allocLocal(Type.i32);
4592 const elem_ty = ty_pl.ty.toType().childType(zcu);4653 defer lhs_null.free(cg);
4593 const elem_size = elem_ty.abiSize(zcu);
45944654
4595 const ptr = try cg.resolveInst(bin_op.lhs);4655 try cg.startBlock(.block, .empty);
4596 const index = try cg.resolveInst(bin_op.rhs);
45974656
4598 // load pointer onto the stack4657 try cg.addImm32(if (op == .eq) 0 else 1);
4599 if (ptr_ty.isSlice(zcu)) {4658 try cg.addLocal(.local_set, result.local.value);
4600 _ = try cg.load(ptr, Type.usize, 0);
4601 } else {
4602 try cg.lowerToStack(ptr);
4603 }
46044659
4605 // calculate index into ptr4660 _ = try cg.isNull(lhs, operand_ty, .i32_eq);
4606 try cg.emitWValue(index);4661 try cg.addLocal(.local_tee, lhs_null.local.value);
4607 try cg.addImm32(@intCast(elem_size));4662 _ = try cg.isNull(rhs, operand_ty, .i32_eq);
4608 try cg.addTag(.i32_mul);4663 try cg.addTag(.i32_ne);
4609 try cg.addTag(.i32_add);4664 try cg.addLabel(.br_if, 0);
46104665
4611 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });4666 try cg.addImm32(if (op == .eq) 1 else 0);
4612}4667 try cg.addLocal(.local_set, result.local.value);
46134668
4614fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: Op) InnerError!void {4669 try cg.addLocal(.local_get, lhs_null.local.value);
4615 const zcu = cg.pt.zcu;4670 try cg.addLabel(.br_if, 0);
4616 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4617 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
46184671
4619 const ptr = try cg.resolveInst(bin_op.lhs);4672 _ = try cg.load(lhs, payload_ty, 0);
4620 const offset = try cg.resolveInst(bin_op.rhs);4673 _ = try cg.load(rhs, payload_ty, 0);
4621 const ptr_ty = cg.typeOf(bin_op.lhs);
4622 const pointee_ty = switch (ptr_ty.ptrSize(zcu)) {
4623 .one => ptr_ty.childType(zcu).childType(zcu), // ptr to array, so get array element type
4624 else => ptr_ty.childType(zcu),
4625 };
46264674
4627 const valtype = typeToValtype(Type.usize, zcu, cg.target);4675 if (payload_ty.isAnyFloat()) {
4628 const mul_opcode = buildOpcode(.{ .valtype1 = valtype, .op = .mul });4676 _ = try cg.floatCmp(.fromType(cg, payload_ty), op, .stack, .stack);
4629 const bin_opcode = buildOpcode(.{ .valtype1 = valtype, .op = op });4677 } else {
4678 _ = try cg.intCmp(.fromType(cg, payload_ty), op, .stack, .stack);
4679 }
46304680
4631 try cg.lowerToStack(ptr);4681 try cg.addLocal(.local_set, result.local.value);
4632 try cg.emitWValue(offset);4682 try cg.endBlock();
4633 try cg.addImm32(@intCast(pointee_ty.abiSize(zcu)));
4634 try cg.addTag(Mir.Inst.Tag.fromOpcode(mul_opcode));
4635 try cg.addTag(Mir.Inst.Tag.fromOpcode(bin_opcode));
46364683
4637 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });4684 try cg.addLocal(.local_get, result.local.value);
4638}4685 try cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
4686 } else {
4687 const result = try cg.intCmp(.fromType(cg, operand_ty), op, lhs, rhs);
4688 try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
4689 }
4690 } else if (type_tag == .float) {
4691 const result = try cg.floatCmp(.fromType(cg, operand_ty), op, lhs, rhs);
4692 try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
4693 } else {
4694 const result = try cg.intCmp(.fromType(cg, operand_ty), op, lhs, rhs);
4695 try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
4696 }
4697}
4698
4699fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue, rhs: WValue) InnerError!WValue {
4700 switch (ty.bits) {
4701 0 => unreachable,
4702 1...32 => {
4703 // lhs or rhs could be stack pointers
4704 try cg.lowerToStack(lhs);
4705 try cg.lowerToStack(rhs);
4706 const opcode: Mir.Inst.Tag = switch (op) {
4707 .eq => .i32_eq,
4708 .neq => .i32_ne,
4709 .lt => if (ty.is_signed) .i32_lt_s else .i32_lt_u,
4710 .lte => if (ty.is_signed) .i32_le_s else .i32_le_u,
4711 .gte => if (ty.is_signed) .i32_ge_s else .i32_ge_u,
4712 .gt => if (ty.is_signed) .i32_gt_s else .i32_gt_u,
4713 };
4714 try cg.addTag(opcode);
4715 return .stack;
4716 },
4717 33...64 => {
4718 // lhs or rhs could be stack pointers
4719 try cg.lowerToStack(lhs);
4720 try cg.lowerToStack(rhs);
4721 const opcode: Mir.Inst.Tag = switch (op) {
4722 .eq => .i64_eq,
4723 .neq => .i64_ne,
4724 .lt => if (ty.is_signed) .i64_lt_s else .i64_lt_u,
4725 .lte => if (ty.is_signed) .i64_le_s else .i64_le_u,
4726 .gte => if (ty.is_signed) .i64_ge_s else .i64_ge_u,
4727 .gt => if (ty.is_signed) .i64_gt_s else .i64_gt_u,
4728 };
4729 try cg.addTag(opcode);
4730 return .stack;
4731 },
4732 65...128 => {
4733 var lhs_msb = try (try cg.load(lhs, Type.u64, 8)).toLocal(cg, Type.u64);
4734 defer lhs_msb.free(cg);
4735 var rhs_msb = try (try cg.load(rhs, Type.u64, 8)).toLocal(cg, Type.u64);
4736 defer rhs_msb.free(cg);
46394737
4640fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {4738 switch (op) {
4641 const zcu = cg.pt.zcu;4739 .eq, .neq => {
4642 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;4740 const xor_high = try cg.intXor(.u64, lhs_msb, rhs_msb);
4741 const lhs_lsb = try cg.load(lhs, Type.u64, 0);
4742 const rhs_lsb = try cg.load(rhs, Type.u64, 0);
4743 const xor_low = try cg.intXor(.u64, lhs_lsb, rhs_lsb);
4744 const or_result = try cg.intOr(.u64, xor_high, xor_low);
4745
4746 switch (op) {
4747 .eq => return cg.intCmp(.u64, .eq, or_result, .{ .imm64 = 0 }),
4748 .neq => return cg.intCmp(.u64, .neq, or_result, .{ .imm64 = 0 }),
4749 else => unreachable,
4750 }
4751 },
4752 else => {
4753 const word_int_ty: IntType = if (ty.is_signed) .i64 else .u64;
46434754
4644 const ptr = try cg.resolveInst(bin_op.lhs);4755 const lhs_lsb = try cg.load(lhs, Type.u64, 0);
4645 const ptr_ty = cg.typeOf(bin_op.lhs);4756 const rhs_lsb = try cg.load(rhs, Type.u64, 0);
4646 const value = try cg.resolveInst(bin_op.rhs);
4647 const len = switch (ptr_ty.ptrSize(zcu)) {
4648 .slice => try cg.sliceLen(ptr),
4649 .one => @as(WValue, .{ .imm32 = @as(u32, @intCast(ptr_ty.childType(zcu).arrayLen(zcu))) }),
4650 .c, .many => unreachable,
4651 };
46524757
4653 const elem_ty = if (ptr_ty.ptrSize(zcu) == .one)4758 // leave values on stack for 'select'
4654 ptr_ty.childType(zcu).childType(zcu)4759 _ = try cg.intCmp(.u64, op, lhs_lsb, rhs_lsb);
4655 else4760 _ = try cg.intCmp(word_int_ty, op, lhs_msb, rhs_msb);
4656 ptr_ty.childType(zcu);4761 _ = try cg.intCmp(word_int_ty, .eq, lhs_msb, rhs_msb);
4762 try cg.addTag(.select);
4763 },
4764 }
46574765
4658 if (!safety and bin_op.rhs == .undef) {4766 return .stack;
4659 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });4767 },
4768 else => return cg.fail("TODO: Support intCmp for integer bitsize: {d}", .{ty.bits}),
4660 }4769 }
4661
4662 const dst_ptr = try cg.sliceOrArrayPtr(ptr, ptr_ty);
4663 try cg.memset(elem_ty, dst_ptr, len, value);
4664
4665 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
4666}4770}
46674771
4668/// Sets a region of memory at `ptr` to the value of `value`4772fn floatCmp(cg: *CodeGen, ty: FloatType, op: std.math.CompareOperator, lhs: WValue, rhs: WValue) InnerError!WValue {
4669/// When the user has enabled the bulk_memory feature, we lower4773 switch (ty) {
4670/// this to wasm's memset instruction. When the feature is not present,4774 .f16 => {
4671/// we implement it manually.4775 _ = try cg.floatExtendCast(.f32, .f16, lhs);
4672fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) InnerError!void {4776 _ = try cg.floatExtendCast(.f32, .f16, rhs);
4673 const zcu = cg.pt.zcu;4777 try cg.addTag(switch (op) {
4674 const abi_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));4778 .eq => .f32_eq,
4779 .neq => .f32_ne,
4780 .lt => .f32_lt,
4781 .lte => .f32_le,
4782 .gte => .f32_ge,
4783 .gt => .f32_gt,
4784 });
4785 return .stack;
4786 },
4787 .f32 => {
4788 try cg.emitWValue(lhs);
4789 try cg.emitWValue(rhs);
4790 try cg.addTag(switch (op) {
4791 .eq => .f32_eq,
4792 .neq => .f32_ne,
4793 .lt => .f32_lt,
4794 .lte => .f32_le,
4795 .gte => .f32_ge,
4796 .gt => .f32_gt,
4797 });
4798 return .stack;
4799 },
4800 .f64 => {
4801 try cg.emitWValue(lhs);
4802 try cg.emitWValue(rhs);
4803 try cg.addTag(switch (op) {
4804 .eq => .f64_eq,
4805 .neq => .f64_ne,
4806 .lt => .f64_lt,
4807 .lte => .f64_le,
4808 .gte => .f64_ge,
4809 .gt => .f64_gt,
4810 });
4811 return .stack;
4812 },
4813 .f80 => {
4814 const intrinsic: Mir.Intrinsic = switch (op) {
4815 .lt => .__ltxf2,
4816 .lte => .__lexf2,
4817 .eq => .__eqxf2,
4818 .neq => .__nexf2,
4819 .gte => .__gexf2,
4820 .gt => .__gtxf2,
4821 };
4822 const result = try cg.callIntrinsic(intrinsic, &.{ .f80_type, .f80_type }, Type.bool, &.{ lhs, rhs });
4823 return cg.intCmp(.i32, op, result, .{ .imm32 = 0 });
4824 },
4825 .f128 => {
4826 const intrinsic: Mir.Intrinsic = switch (op) {
4827 .lt => .__lttf2,
4828 .lte => .__letf2,
4829 .eq => .__eqtf2,
4830 .neq => .__netf2,
4831 .gte => .__getf2,
4832 .gt => .__gttf2,
4833 };
4834 const result = try cg.callIntrinsic(intrinsic, &.{ .f128_type, .f128_type }, Type.bool, &.{ lhs, rhs });
4835 return cg.intCmp(.i32, op, result, .{ .imm32 = 0 });
4836 },
4837 }
4838}
46754839
4676 // When bulk_memory is enabled, we lower it to wasm's memset instruction.4840fn airCmpVector(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4677 // If not, we lower it ourselves.4841 _ = inst;
4678 if (cg.target.cpu.has(.wasm, .bulk_memory) and abi_size == 1) {4842 return cg.fail("TODO implement airCmpVector for wasm", .{});
4679 const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0);4843}
46804844
4681 if (!len0_ok) {4845fn airCmpLtErrorsLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4682 try cg.startBlock(.block, .empty);4846 const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4847 const operand = try cg.resolveInst(un_op);
46834848
4684 // Even if `len` is zero, the spec requires an implementation to trap if `ptr + len` is4849 try cg.emitWValue(operand);
4685 // out of memory bounds. This can easily happen in Zig in a case such as:4850 const pt = cg.pt;
4686 //4851 const err_int_ty = try pt.errorIntType();
4687 // const ptr: [*]u8 = undefined;4852 try cg.addTag(.errors_len);
4688 // var len: usize = runtime_zero();4853 const result = try cg.intCmp(.fromType(cg, err_int_ty), .lt, .stack, .stack);
4689 // @memset(ptr[0..len], 42);
4690 //
4691 // So explicitly avoid using `memory.fill` in the `len == 0` case. Lovely design.
4692 try cg.emitWValue(len);
4693 try cg.addTag(.i32_eqz);
4694 try cg.addLabel(.br_if, 0);
4695 }
46964854
4697 try cg.lowerToStack(ptr);4855 return cg.finishAir(inst, result, &.{un_op});
4698 try cg.emitWValue(value);4856}
4699 try cg.emitWValue(len);
4700 try cg.addExtended(.memory_fill);
47014857
4702 if (!len0_ok) {4858fn airBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4703 try cg.endBlock();4859 const br = cg.air.instructions.items(.data)[@intFromEnum(inst)].br;
4704 }4860 const block = cg.blocks.get(br.block_inst).?;
47054861
4706 return;4862 // if operand has codegen bits we should break with a value
4863 if (block.value != .none) {
4864 const operand = try cg.resolveInst(br.operand);
4865 try cg.lowerToStack(operand);
4866 try cg.addLocal(.local_set, block.value.local.value);
4707 }4867 }
47084868
4709 const final_len: WValue = switch (len) {4869 // We map every block to its block index.
4710 .imm32 => |val| .{ .imm32 = val * abi_size },4870 // We then determine how far we have to jump to it by subtracting it from current block depth
4711 .imm64 => |val| .{ .imm64 = val * abi_size },4871 const idx: u32 = cg.block_depth - block.label;
4712 else => if (abi_size != 1) blk: {4872 try cg.addLabel(.br, idx);
4713 const new_len = try cg.ensureAllocLocal(Type.usize);
4714 try cg.emitWValue(len);
4715 switch (cg.ptr_size) {
4716 .wasm32 => {
4717 try cg.emitWValue(.{ .imm32 = abi_size });
4718 try cg.addTag(.i32_mul);
4719 },
4720 .wasm64 => {
4721 try cg.emitWValue(.{ .imm64 = abi_size });
4722 try cg.addTag(.i64_mul);
4723 },
4724 }
4725 try cg.addLocal(.local_set, new_len.local.value);
4726 break :blk new_len;
4727 } else len,
4728 };
4729
4730 var end_ptr = try cg.allocLocal(Type.usize);
4731 defer end_ptr.free(cg);
4732 var new_ptr = try cg.buildPointerOffset(ptr, 0, .new);
4733 defer new_ptr.free(cg);
47344873
4735 // get the loop conditional: if current pointer address equals final pointer's address4874 return cg.finishAir(inst, .none, &.{br.operand});
4736 try cg.lowerToStack(ptr);4875}
4737 try cg.emitWValue(final_len);
4738 switch (cg.ptr_size) {
4739 .wasm32 => try cg.addTag(.i32_add),
4740 .wasm64 => try cg.addTag(.i64_add),
4741 }
4742 try cg.addLocal(.local_set, end_ptr.local.value);
47434876
4744 // outer block to jump to when loop is done4877fn airRepeat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4745 try cg.startBlock(.block, .empty);4878 const repeat = cg.air.instructions.items(.data)[@intFromEnum(inst)].repeat;
4746 try cg.startBlock(.loop, .empty);4879 const loop_label = cg.loops.get(repeat.loop_inst).?;
47474880
4748 // check for condition for loop end4881 const idx: u32 = cg.block_depth - loop_label;
4749 try cg.emitWValue(new_ptr);4882 try cg.addLabel(.br, idx);
4750 try cg.emitWValue(end_ptr);
4751 switch (cg.ptr_size) {
4752 .wasm32 => try cg.addTag(.i32_eq),
4753 .wasm64 => try cg.addTag(.i64_eq),
4754 }
4755 try cg.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
47564883
4757 // store the value at the current position of the pointer4884 return cg.finishAir(inst, .none, &.{});
4758 try cg.store(new_ptr, value, elem_ty, 0);4885}
47594886
4760 // move the pointer to the next element4887fn airTrap(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4761 try cg.emitWValue(new_ptr);4888 try cg.addTag(.@"unreachable");
4762 switch (cg.ptr_size) {4889 return cg.finishAir(inst, .none, &.{});
4763 .wasm32 => {4890}
4764 try cg.emitWValue(.{ .imm32 = abi_size });
4765 try cg.addTag(.i32_add);
4766 },
4767 .wasm64 => {
4768 try cg.emitWValue(.{ .imm64 = abi_size });
4769 try cg.addTag(.i64_add);
4770 },
4771 }
4772 try cg.addLocal(.local_set, new_ptr.local.value);
47734891
4774 // end of loop4892fn airBreakpoint(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4775 try cg.addLabel(.br, 0); // jump to start of loop4893 // unsupported by wasm itfunc. Can be implemented once we support DWARF
4776 try cg.endBlock();4894 // for wasm
4777 try cg.endBlock();4895 try cg.addTag(.@"unreachable");
4896 return cg.finishAir(inst, .none, &.{});
4778}4897}
47794898
4780fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {4899fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4781 const zcu = cg.pt.zcu;4900 try cg.addTag(.@"unreachable");
4782 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;4901 return cg.finishAir(inst, .none, &.{});
4902}
47834903
4784 const array_ty = cg.typeOf(bin_op.lhs);4904fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4785 const array = try cg.resolveInst(bin_op.lhs);4905 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4786 const index = try cg.resolveInst(bin_op.rhs);4906 const operand = try cg.resolveInst(ty_op.operand);
4787 const elem_ty = array_ty.childType(zcu);4907 const dest_ty = cg.typeOfIndex(inst);
4788 const elem_size = elem_ty.abiSize(zcu);4908 const src_ty = cg.typeOf(ty_op.operand);
47894909
4790 if (isByRef(array_ty, zcu, cg.target)) {4910 const result = (try cg.bitcast(dest_ty, src_ty, operand)) orelse cg.reuseOperand(ty_op.operand, operand);
4791 try cg.lowerToStack(array);
4792 try cg.emitWValue(index);
4793 try cg.addImm32(@intCast(elem_size));
4794 try cg.addTag(.i32_mul);
4795 try cg.addTag(.i32_add);
4796 } else {
4797 assert(array_ty.zigTypeTag(zcu) == .vector);
47984911
4799 switch (index) {4912 return cg.finishAir(inst, result, &.{ty_op.operand});
4800 inline .imm32, .imm64 => |lane| {4913}
4801 const opcode: std.wasm.SimdOpcode = switch (elem_ty.bitSize(zcu)) {
4802 8 => if (elem_ty.isSignedInt(zcu)) .i8x16_extract_lane_s else .i8x16_extract_lane_u,
4803 16 => if (elem_ty.isSignedInt(zcu)) .i16x8_extract_lane_s else .i16x8_extract_lane_u,
4804 32 => if (elem_ty.isInt(zcu)) .i32x4_extract_lane else .f32x4_extract_lane,
4805 64 => if (elem_ty.isInt(zcu)) .i64x2_extract_lane else .f64x2_extract_lane,
4806 else => unreachable,
4807 };
48084914
4809 var operands = [_]u32{ @intFromEnum(opcode), @as(u8, @intCast(lane)) };4915fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerError!?WValue {
4916 const zcu = cg.pt.zcu;
4917 const bit_size = src_ty.bitSize(zcu);
4918 const needs_wrapping = (src_ty.isSignedInt(zcu) != dest_ty.isSignedInt(zcu)) and
4919 bit_size != 32 and bit_size != 64 and bit_size != 128;
48104920
4811 try cg.emitWValue(array);4921 if (src_ty.isAnyFloat() or dest_ty.isAnyFloat()) {
4922 if (dest_ty.ip_index == .f16_type or src_ty.ip_index == .f16_type) return null;
4923 if (dest_ty.bitSize(zcu) > 64) return null;
4924 assert((dest_ty.isInt(zcu) and src_ty.isAnyFloat()) or (dest_ty.isAnyFloat() and src_ty.isInt(zcu)));
4925
4926 const dest_valtype = typeToValtype(dest_ty, zcu, cg.target);
4927 const opcode: Mir.Inst.Tag = switch (dest_valtype) {
4928 .i32 => .i32_reinterpret_f32,
4929 .i64 => .i64_reinterpret_f64,
4930 .f32 => .f32_reinterpret_i32,
4931 .f64 => .f64_reinterpret_i64,
4932 else => unreachable,
4933 };
48124934
4813 const extra_index: u32 = @intCast(cg.mir_extra.items.len);4935 try cg.emitWValue(operand);
4814 try cg.mir_extra.appendSlice(cg.gpa, &operands);4936 try cg.addTag(opcode);
4815 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });4937 return .stack;
4938 }
48164939
4817 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });4940 if (isByRef(src_ty, zcu, cg.target) and !isByRef(dest_ty, zcu, cg.target)) {
4818 },4941 const loaded_memory = try cg.load(operand, dest_ty, 0);
4819 else => {4942 if (needs_wrapping) {
4820 const stack_vec = try cg.allocStack(array_ty);4943 const int_ty: IntType = .fromType(cg, dest_ty);
4821 try cg.store(stack_vec, array, array_ty, 0);4944 return try cg.intWrap(int_ty, loaded_memory);
4945 } else {
4946 return loaded_memory;
4947 }
4948 }
48224949
4823 // Is a non-unrolled vector (v128)4950 if (!isByRef(src_ty, zcu, cg.target) and isByRef(dest_ty, zcu, cg.target)) {
4824 try cg.lowerToStack(stack_vec);4951 const stack_memory = try cg.allocStack(dest_ty);
4825 try cg.emitWValue(index);4952 try cg.store(stack_memory, operand, src_ty, 0);
4826 try cg.addImm32(@intCast(elem_size));4953 if (needs_wrapping) {
4827 try cg.addTag(.i32_mul);4954 const int_ty: IntType = .fromType(cg, dest_ty);
4828 try cg.addTag(.i32_add);4955 return try cg.intWrap(int_ty, stack_memory);
4829 },4956 } else {
4957 return stack_memory;
4830 }4958 }
4831 }4959 }
48324960
4833 const elem_result = if (isByRef(elem_ty, zcu, cg.target))4961 if (needs_wrapping) {
4834 .stack4962 const int_ty: IntType = .fromType(cg, dest_ty);
4835 else4963 return try cg.intWrap(int_ty, operand);
4836 try cg.load(.stack, elem_ty, 0);4964 }
48374965
4838 return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });4966 return switch (operand) {
4967 // for stack offset, return a pointer to this offset.
4968 .stack_offset => try cg.buildPointerOffset(operand, 0, .new),
4969 else => null, // caller should use cg.reuseOperand, if returnes for AIR
4970 };
4971}
4972
4973fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4974 const zcu = cg.pt.zcu;
4975 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4976 const extra = cg.air.extraData(Air.StructField, ty_pl.payload);
4977
4978 const struct_ptr = try cg.resolveInst(extra.data.struct_operand);
4979 const struct_ptr_ty = cg.typeOf(extra.data.struct_operand);
4980 const struct_ty = struct_ptr_ty.childType(zcu);
4981 const result = try cg.structFieldPtr(inst, extra.data.struct_operand, struct_ptr, struct_ptr_ty, struct_ty, extra.data.field_index);
4982 return cg.finishAir(inst, result, &.{extra.data.struct_operand});
4839}4983}
48404984
4841fn airIntFromFloat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {4985fn airStructFieldPtrIndex(cg: *CodeGen, inst: Air.Inst.Index, index: u32) InnerError!void {
4842 const zcu = cg.pt.zcu;4986 const zcu = cg.pt.zcu;
4843 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4987 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4988 const struct_ptr = try cg.resolveInst(ty_op.operand);
4989 const struct_ptr_ty = cg.typeOf(ty_op.operand);
4990 const struct_ty = struct_ptr_ty.childType(zcu);
48444991
4845 const operand = try cg.resolveInst(ty_op.operand);4992 const result = try cg.structFieldPtr(inst, ty_op.operand, struct_ptr, struct_ptr_ty, struct_ty, index);
4846 const op_ty = cg.typeOf(ty_op.operand);4993 return cg.finishAir(inst, result, &.{ty_op.operand});
4847 const op_bits = op_ty.floatBits(cg.target);4994}
48484995
4849 const dest_ty = cg.typeOfIndex(inst);4996fn structFieldPtr(
4850 const dest_info = dest_ty.intInfo(zcu);4997 cg: *CodeGen,
48514998 inst: Air.Inst.Index,
4852 if (dest_info.bits > 128) {4999 ref: Air.Inst.Ref,
4853 return cg.fail("TODO: intFromFloat for integers/floats with bitsize {}", .{dest_info.bits});5000 struct_ptr: WValue,
4854 }5001 struct_ptr_ty: Type,
48555002 struct_ty: Type,
4856 if ((op_bits != 32 and op_bits != 64) or dest_info.bits > 64) {5003 index: u32,
4857 const dest_bitsize = if (dest_info.bits <= 32) 32 else std.math.ceilPowerOfTwoAssert(u16, dest_info.bits);5004) InnerError!WValue {
48585005 const pt = cg.pt;
4859 const intrinsic = switch (dest_info.signedness) {5006 const zcu = pt.zcu;
4860 inline .signed, .unsigned => |ct_s| switch (op_bits) {5007 const result_ty = cg.typeOfIndex(inst);
4861 inline 16, 32, 64, 80, 128 => |ct_op_bits| switch (dest_bitsize) {5008 const struct_ptr_ty_info = struct_ptr_ty.ptrInfo(zcu);
4862 inline 32, 64, 128 => |ct_dest_bits| @field(5009
4863 Mir.Intrinsic,5010 const offset = switch (struct_ty.containerLayout(zcu)) {
4864 "__fix" ++ switch (ct_s) {5011 .@"packed" => switch (struct_ty.zigTypeTag(zcu)) {
4865 .signed => "",5012 .@"struct" => offset: {
4866 .unsigned => "uns",5013 if (result_ty.ptrInfo(zcu).packed_offset.host_size != 0) {
4867 } ++5014 break :offset @as(u32, 0);
4868 compilerRtFloatAbbrev(ct_op_bits) ++ "f" ++5015 }
4869 compilerRtIntAbbrev(ct_dest_bits) ++ "i",5016 const struct_type = zcu.typeToStruct(struct_ty).?;
4870 ),5017 break :offset @divExact(zcu.structPackedFieldBitOffset(struct_type, index) + struct_ptr_ty_info.packed_offset.bit_offset, 8);
4871 else => unreachable,
4872 },
4873 else => unreachable,
4874 },5018 },
4875 };5019 .@"union" => 0,
4876 const result = try cg.callIntrinsic(intrinsic, &.{op_ty.ip_index}, dest_ty, &.{operand});5020 else => unreachable,
4877 return cg.finishAir(inst, result, &.{ty_op.operand});5021 },
5022 else => struct_ty.structFieldOffset(index, zcu),
5023 };
5024 // save a load and store when we can simply reuse the operand
5025 if (offset == 0) {
5026 return cg.reuseOperand(ref, struct_ptr);
5027 }
5028 switch (struct_ptr) {
5029 .stack_offset => |stack_offset| {
5030 return .{ .stack_offset = .{ .value = stack_offset.value + @as(u32, @intCast(offset)), .references = 1 } };
5031 },
5032 else => return cg.buildPointerOffset(struct_ptr, offset, .new),
4878 }5033 }
5034}
48795035
4880 try cg.emitWValue(operand);5036fn airStructFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4881 const op = buildOpcode(.{5037 const pt = cg.pt;
4882 .op = .trunc,5038 const zcu = pt.zcu;
4883 .valtype1 = typeToValtype(dest_ty, zcu, cg.target),5039 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4884 .valtype2 = typeToValtype(op_ty, zcu, cg.target),5040 const struct_field = cg.air.extraData(Air.StructField, ty_pl.payload).data;
4885 .signedness = dest_info.signedness,5041
4886 });5042 const struct_ty = cg.typeOf(struct_field.struct_operand);
4887 try cg.addTag(Mir.Inst.Tag.fromOpcode(op));5043 const operand = try cg.resolveInst(struct_field.struct_operand);
4888 const result = try cg.wrapOperand(.stack, dest_ty);5044 const field_index = struct_field.field_index;
4889 return cg.finishAir(inst, result, &.{ty_op.operand});5045 const field_ty = struct_ty.fieldType(field_index, zcu);
5046 if (!field_ty.hasRuntimeBits(zcu)) return cg.finishAir(inst, .none, &.{struct_field.struct_operand});
5047
5048 const result: WValue = switch (struct_ty.containerLayout(zcu)) {
5049 .@"packed" => unreachable, // legalize .expand_packed_struct_field_val
5050 else => result: {
5051 const offset = std.math.cast(u32, struct_ty.structFieldOffset(field_index, zcu)) orelse {
5052 return cg.fail("Field type '{f}' too big to fit into stack frame", .{field_ty.fmt(pt)});
5053 };
5054 if (isByRef(field_ty, zcu, cg.target)) {
5055 switch (operand) {
5056 .stack_offset => |stack_offset| {
5057 break :result .{ .stack_offset = .{ .value = stack_offset.value + offset, .references = 1 } };
5058 },
5059 else => break :result try cg.buildPointerOffset(operand, offset, .new),
5060 }
5061 }
5062 break :result try cg.load(operand, field_ty, offset);
5063 },
5064 };
5065
5066 return cg.finishAir(inst, result, &.{struct_field.struct_operand});
4890}5067}
48915068
4892fn airFloatFromInt(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5069fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) InnerError!void {
4893 const zcu = cg.pt.zcu;5070 const pt = cg.pt;
4894 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5071 const zcu = pt.zcu;
48955072
4896 const operand = try cg.resolveInst(ty_op.operand);5073 const switch_br = cg.air.unwrapSwitch(inst);
4897 const op_ty = cg.typeOf(ty_op.operand);5074 const target_ty = cg.typeOf(switch_br.operand);
4898 const op_info = op_ty.intInfo(zcu);
48995075
4900 const dest_ty = cg.typeOfIndex(inst);5076 assert(target_ty.hasRuntimeBits(zcu));
4901 const dest_bits = dest_ty.floatBits(cg.target);
4902
4903 if (op_info.bits > 128) {
4904 return cg.fail("TODO: floatFromInt for integers/floats with bitsize {d} bits", .{op_info.bits});
4905 }
4906
4907 if (op_info.bits > 64 or (dest_bits > 64 or dest_bits < 32)) {
4908 const op_bitsize = if (op_info.bits <= 32) 32 else std.math.ceilPowerOfTwoAssert(u16, op_info.bits);
4909
4910 const intrinsic = switch (op_info.signedness) {
4911 inline .signed, .unsigned => |ct_s| switch (op_bitsize) {
4912 inline 32, 64, 128 => |ct_int_bits| switch (dest_bits) {
4913 inline 16, 32, 64, 80, 128 => |ct_float_bits| @field(
4914 Mir.Intrinsic,
4915 "__float" ++ switch (ct_s) {
4916 .signed => "",
4917 .unsigned => "un",
4918 } ++
4919 compilerRtIntAbbrev(ct_int_bits) ++ "i" ++
4920 compilerRtFloatAbbrev(ct_float_bits) ++ "f",
4921 ),
4922 else => unreachable,
4923 },
4924 else => unreachable,
4925 },
4926 };
49275077
4928 const result = try cg.callIntrinsic(intrinsic, &.{op_ty.ip_index}, dest_ty, &.{operand});5078 // swap target value with placeholder local, for dispatching
4929 return cg.finishAir(inst, result, &.{ty_op.operand});5079 const target = if (is_dispatch_loop) target: {
4930 }5080 const initial_target = try cg.resolveInst(switch_br.operand);
5081 const target: WValue = try cg.allocLocal(target_ty);
5082 try cg.lowerToStack(initial_target);
5083 try cg.addLocal(.local_set, target.local.value);
49315084
4932 try cg.emitWValue(operand);5085 try cg.startBlock(.loop, .empty); // dispatch loop start
4933 const op = buildOpcode(.{5086 try cg.blocks.putNoClobber(cg.gpa, inst, .{
4934 .op = .convert,5087 .label = cg.block_depth,
4935 .valtype1 = typeToValtype(dest_ty, zcu, cg.target),5088 .value = target,
4936 .valtype2 = typeToValtype(op_ty, zcu, cg.target),5089 });
4937 .signedness = op_info.signedness,
4938 });
4939 try cg.addTag(Mir.Inst.Tag.fromOpcode(op));
49405090
4941 return cg.finishAir(inst, .stack, &.{ty_op.operand});5091 break :target target;
4942}5092 } else try cg.resolveInst(switch_br.operand);
49435093
4944fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5094 const liveness = try cg.liveness.getSwitchBr(cg.gpa, inst, switch_br.cases_len + 1);
4945 const zcu = cg.pt.zcu;5095 defer cg.gpa.free(liveness.deaths);
4946 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4947 const operand = try cg.resolveInst(ty_op.operand);
4948 const ty = cg.typeOfIndex(inst);
4949 const elem_ty = ty.childType(zcu);
49505096
4951 if (determineSimdStoreStrategy(ty, zcu, cg.target) == .direct) blk: {5097 const has_else_body = switch_br.else_body_len != 0;
4952 switch (operand) {5098 const branch_count = switch_br.cases_len + 1; // if else branch is missing, we trap when failing all conditions
4953 // when the operand lives in the linear memory section, we can directly5099 try cg.branches.ensureUnusedCapacity(cg.gpa, switch_br.cases_len + @intFromBool(has_else_body));
4954 // load and splat the value at once. Meaning we do not first have to load5100
4955 // the scalar value onto the stack.5101 if (switch_br.cases_len == 0) {
4956 .stack_offset, .nav_ref, .uav_ref => {5102 assert(has_else_body);
4957 const opcode = switch (elem_ty.bitSize(zcu)) {5103
4958 8 => @intFromEnum(std.wasm.SimdOpcode.v128_load8_splat),5104 var it = switch_br.iterateCases();
4959 16 => @intFromEnum(std.wasm.SimdOpcode.v128_load16_splat),5105 const else_body = it.elseBody();
4960 32 => @intFromEnum(std.wasm.SimdOpcode.v128_load32_splat),5106
4961 64 => @intFromEnum(std.wasm.SimdOpcode.v128_load64_splat),5107 cg.branches.appendAssumeCapacity(.{});
4962 else => break :blk, // Cannot make use of simd-instructions5108 const else_deaths = liveness.deaths.len - 1;
4963 };5109 try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.deaths[else_deaths].len);
4964 try cg.emitWValue(operand);5110 defer {
4965 const extra_index: u32 = @intCast(cg.mir_extra.items.len);5111 var else_branch = cg.branches.pop().?;
4966 // stores as := opcode, offset, alignment (opcode::memarg)5112 else_branch.deinit(cg.gpa);
4967 try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
4968 opcode,
4969 operand.offset(),
4970 @intCast(elem_ty.abiAlignment(zcu).toByteUnits().?),
4971 });
4972 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
4973 return cg.finishAir(inst, .stack, &.{ty_op.operand});
4974 },
4975 .local => {
4976 const opcode = switch (elem_ty.bitSize(zcu)) {
4977 8 => @intFromEnum(std.wasm.SimdOpcode.i8x16_splat),
4978 16 => @intFromEnum(std.wasm.SimdOpcode.i16x8_splat),
4979 32 => if (elem_ty.isInt(zcu)) @intFromEnum(std.wasm.SimdOpcode.i32x4_splat) else @intFromEnum(std.wasm.SimdOpcode.f32x4_splat),
4980 64 => if (elem_ty.isInt(zcu)) @intFromEnum(std.wasm.SimdOpcode.i64x2_splat) else @intFromEnum(std.wasm.SimdOpcode.f64x2_splat),
4981 else => break :blk, // Cannot make use of simd-instructions
4982 };
4983 try cg.emitWValue(operand);
4984 const extra_index: u32 = @intCast(cg.mir_extra.items.len);
4985 try cg.mir_extra.append(cg.gpa, opcode);
4986 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
4987 return cg.finishAir(inst, .stack, &.{ty_op.operand});
4988 },
4989 else => unreachable,
4990 }5113 }
4991 }5114 try cg.genBody(else_body);
4992 const elem_size = elem_ty.bitSize(zcu);
4993 const vector_len = @as(usize, @intCast(ty.vectorLen(zcu)));
4994 if ((!std.math.isPowerOfTwo(elem_size) or elem_size % 8 != 0) and vector_len > 1) {
4995 return cg.fail("TODO: WebAssembly `@splat` for arbitrary element bitsize {d}", .{elem_size});
4996 }
49975115
4998 const result = try cg.allocStack(ty);5116 if (is_dispatch_loop) {
4999 const elem_byte_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));5117 try cg.endBlock(); // dispatch loop end
5000 var index: usize = 0;5118 }
5001 var offset: u32 = 0;5119 return cg.finishAir(inst, .none, &.{});
5002 while (index < vector_len) : (index += 1) {
5003 try cg.store(result, operand, elem_ty, offset);
5004 offset += elem_byte_size;
5005 }5120 }
50065121
5007 return cg.finishAir(inst, result, &.{ty_op.operand});5122 var min: ?Value = null;
5008}5123 var max: ?Value = null;
5124 var branching_size: u32 = 0; // single item +1, range +2
50095125
5010fn airSelect(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5126 {
5011 const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;5127 var cases_it = switch_br.iterateCases();
5012 const operand = try cg.resolveInst(pl_op.operand);5128 while (cases_it.next()) |case| {
5129 for (case.items) |item| {
5130 const val = Value.fromInterned(item.toInterned().?);
5131 if (min == null or val.compareHetero(.lt, min.?, zcu)) min = val;
5132 if (max == null or val.compareHetero(.gt, max.?, zcu)) max = val;
5133 branching_size += 1;
5134 }
5135 for (case.ranges) |range| {
5136 const low = Value.fromInterned(range[0].toInterned().?);
5137 if (min == null or low.compareHetero(.lt, min.?, zcu)) min = low;
5138 const high = Value.fromInterned(range[1].toInterned().?);
5139 if (max == null or high.compareHetero(.gt, max.?, zcu)) max = high;
5140 branching_size += 2;
5141 }
5142 }
5143 }
50135144
5014 _ = operand;5145 var min_space: Value.BigIntSpace = undefined;
5015 return cg.fail("TODO: Implement wasm airSelect", .{});5146 const min_bigint = min.?.toBigInt(&min_space, zcu);
5016}5147 var max_space: Value.BigIntSpace = undefined;
5148 const max_bigint = max.?.toBigInt(&max_space, zcu);
5149 const limbs = try cg.gpa.alloc(
5150 std.math.big.Limb,
5151 @max(min_bigint.limbs.len, max_bigint.limbs.len) + 1,
5152 );
5153 defer cg.gpa.free(limbs);
50175154
5018fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5155 const width_maybe: ?u32 = width: {
5019 const pt = cg.pt;5156 var width_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
5020 const zcu = pt.zcu;5157 width_bigint.sub(max_bigint, min_bigint);
5158 width_bigint.addScalar(width_bigint.toConst(), 1);
5159 break :width width_bigint.toConst().toInt(u32) catch null;
5160 };
50215161
5022 const unwrapped = cg.air.unwrapShuffleOne(zcu, inst);5162 try cg.startBlock(.block, .empty); // whole switch block start
5023 const result_ty = unwrapped.result_ty;
5024 const mask = unwrapped.mask;
5025 const operand = try cg.resolveInst(unwrapped.operand);
50265163
5027 const elem_ty = result_ty.childType(zcu);5164 for (0..branch_count) |_| {
5028 const elem_size = elem_ty.abiSize(zcu);5165 try cg.startBlock(.block, .empty);
5166 }
50295167
5030 // TODO: this function could have an `i8x16_shuffle` fast path like `airShuffleTwo` if we were5168 // Heuristic on deciding when to use .br_table instead of .br_if jump table
5031 // to lower the comptime-known operands to a non-by-ref vector value.5169 // 1. Differences between lowest and highest values should fit into u32
5170 // 2. .br_table should be applied for "dense" switch, we test it by checking .br_if jumps will need more instructions
5171 // 3. Do not use .br_table for tiny switches
5172 const use_br_table = cond: {
5173 const width = width_maybe orelse break :cond false;
5174 if (width > 2 * branching_size) break :cond false;
5175 if (width < 2 or branch_count < 2) break :cond false;
5176 break :cond true;
5177 };
50325178
5033 // TODO: this is incorrect if either operand or the result is *not* by-ref, which is possible.5179 const int_ty: IntType = .fromType(cg, target_ty);
5034 // I tried to fix it, but I couldn't make much sense of how this backend handles memory.
5035 if (!isByRef(result_ty, zcu, cg.target) or
5036 !isByRef(cg.typeOf(unwrapped.operand), zcu, cg.target)) return cg.fail("TODO: handle mixed by-ref shuffle", .{});
50375180
5038 const dest_alloc = try cg.allocStack(result_ty);5181 if (use_br_table) {
5039 for (mask, 0..) |mask_elem, out_idx| {5182 const width = width_maybe.?;
5040 try cg.emitWValue(dest_alloc);
5041 const elem_val = switch (mask_elem.unwrap()) {
5042 .elem => |idx| try cg.load(operand, elem_ty, @intCast(elem_size * idx)),
5043 .value => |val| try cg.lowerConstant(.fromInterned(val)),
5044 };
5045 try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));
5046 }
5047 return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand});
5048}
50495183
5050fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5184 const br_value_original = try cg.intSub(int_ty, target, try cg.resolveValue(min.?));
5051 const pt = cg.pt;5185 _ = try cg.intCast(.u32, int_ty, br_value_original);
5052 const zcu = pt.zcu;
50535186
5054 const unwrapped = cg.air.unwrapShuffleTwo(zcu, inst);5187 const jump_table: Mir.JumpTable = .{ .length = width + 1 };
5055 const result_ty = unwrapped.result_ty;5188 const table_extra_index = try cg.addExtra(jump_table);
5056 const mask = unwrapped.mask;5189 try cg.addInst(.{ .tag = .br_table, .data = .{ .payload = table_extra_index } });
5057 const operand_a = try cg.resolveInst(unwrapped.operand_a);
5058 const operand_b = try cg.resolveInst(unwrapped.operand_b);
50595190
5060 const a_ty = cg.typeOf(unwrapped.operand_a);5191 const branch_list = try cg.mir_extra.addManyAsSlice(cg.gpa, width + 1);
5061 const b_ty = cg.typeOf(unwrapped.operand_b);5192 @memset(branch_list, branch_count - 1);
5062 const elem_ty = result_ty.childType(zcu);
5063 const elem_size = elem_ty.abiSize(zcu);
50645193
5065 // WASM has `i8x16_shuffle`, which we can apply if the element type bit size is a multiple of 85194 var cases_it = switch_br.iterateCases();
5066 // and the input and output vectors have a bit size of 128 (and are hence not by-ref). Otherwise,5195 while (cases_it.next()) |case| {
5067 // we fall back to a naive loop lowering.5196 for (case.items) |item| {
5068 if (!isByRef(a_ty, zcu, cg.target) and5197 const val = Value.fromInterned(item.toInterned().?);
5069 !isByRef(b_ty, zcu, cg.target) and5198 var val_space: Value.BigIntSpace = undefined;
5070 !isByRef(result_ty, zcu, cg.target) and5199 const val_bigint = val.toBigInt(&val_space, zcu);
5071 elem_ty.bitSize(zcu) % 8 == 0)5200 var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
5072 {5201 index_bigint.sub(val_bigint, min_bigint);
5073 var lane_map: [16]u8 align(4) = undefined;5202 branch_list[index_bigint.toConst().toInt(u32) catch unreachable] = case.idx;
5074 const lanes_per_elem: usize = @intCast(elem_ty.bitSize(zcu) / 8);5203 }
5075 for (mask, 0..) |mask_elem, out_idx| {5204 for (case.ranges) |range| {
5076 const out_first_lane = out_idx * lanes_per_elem;5205 var low_space: Value.BigIntSpace = undefined;
5077 const in_first_lane = switch (mask_elem.unwrap()) {5206 const low_bigint = Value.fromInterned(range[0].toInterned().?).toBigInt(&low_space, zcu);
5078 .a_elem => |i| i * lanes_per_elem,5207 var high_space: Value.BigIntSpace = undefined;
5079 .b_elem => |i| i * lanes_per_elem + 16,5208 const high_bigint = Value.fromInterned(range[1].toInterned().?).toBigInt(&high_space, zcu);
5080 .undef => 0, // doesn't matter5209 var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
5081 };5210 index_bigint.sub(low_bigint, min_bigint);
5082 for (lane_map[out_first_lane..][0..lanes_per_elem], in_first_lane..) |*out, in| {5211 const start = index_bigint.toConst().toInt(u32) catch unreachable;
5083 out.* = @intCast(in);5212 index_bigint.sub(high_bigint, min_bigint);
5213 const end = (index_bigint.toConst().toInt(u32) catch unreachable) + 1;
5214 @memset(branch_list[start..end], case.idx);
5084 }5215 }
5085 }5216 }
5086 try cg.emitWValue(operand_a);5217 } else {
5087 try cg.emitWValue(operand_b);5218 var cases_it = switch_br.iterateCases();
5088 const extra_index: u32 = @intCast(cg.mir_extra.items.len);5219 while (cases_it.next()) |case| {
5089 try cg.mir_extra.appendSlice(cg.gpa, &.{5220 for (case.items) |ref| {
5090 @intFromEnum(std.wasm.SimdOpcode.i8x16_shuffle),5221 const val = try cg.resolveInst(ref);
5091 @bitCast(lane_map[0..4].*),5222 _ = try cg.intCmp(int_ty, .eq, target, val);
5092 @bitCast(lane_map[4..8].*),5223 try cg.addLabel(.br_if, case.idx); // item match found
5093 @bitCast(lane_map[8..12].*),5224 }
5094 @bitCast(lane_map[12..].*),5225 for (case.ranges) |range| {
5095 });5226 const low = try cg.resolveInst(range[0]);
5096 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });5227 const high = try cg.resolveInst(range[1]);
5097 return cg.finishAir(inst, .stack, &.{ unwrapped.operand_a, unwrapped.operand_b });
5098 }
5099
5100 // TODO: this is incorrect if either operand or the result is *not* by-ref, which is possible.
5101 // I tried to fix it, but I couldn't make much sense of how this backend handles memory.
5102 if (!isByRef(result_ty, zcu, cg.target) or
5103 !isByRef(a_ty, zcu, cg.target) or
5104 !isByRef(b_ty, zcu, cg.target)) return cg.fail("TODO: handle mixed by-ref shuffle", .{});
51055228
5106 const dest_alloc = try cg.allocStack(result_ty);5229 const gte = try cg.intCmp(int_ty, .gte, target, low);
5107 for (mask, 0..) |mask_elem, out_idx| {5230 const lte = try cg.intCmp(int_ty, .lte, target, high);
5108 try cg.emitWValue(dest_alloc);5231 _ = try cg.intAnd(.u32, gte, lte);
5109 const elem_val = switch (mask_elem.unwrap()) {5232 try cg.addLabel(.br_if, case.idx); // range match found
5110 .a_elem => |idx| try cg.load(operand_a, elem_ty, @intCast(elem_size * idx)),5233 }
5111 .b_elem => |idx| try cg.load(operand_b, elem_ty, @intCast(elem_size * idx)),5234 }
5112 .undef => try cg.emitUndefined(elem_ty),5235 try cg.addLabel(.br, branch_count - 1);
5113 };
5114 try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));
5115 }5236 }
5116 return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b });
5117}
51185237
5119fn airReduce(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5238 var cases_it = switch_br.iterateCases();
5120 const reduce = cg.air.instructions.items(.data)[@intFromEnum(inst)].reduce;5239 while (cases_it.next()) |case| {
5121 const operand = try cg.resolveInst(reduce.operand);5240 try cg.endBlock();
51225241
5123 _ = operand;5242 cg.branches.appendAssumeCapacity(.{});
5124 return cg.fail("TODO: Implement wasm airReduce", .{});5243 try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.deaths[case.idx].len);
5125}5244 defer {
5245 var case_branch = cg.branches.pop().?;
5246 case_branch.deinit(cg.gpa);
5247 }
5248 try cg.genBody(case.body);
51265249
5127fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5250 try cg.addLabel(.br, branch_count - case.idx - 1); // matching case found and executed => exit switch
5128 const pt = cg.pt;5251 }
5129 const zcu = pt.zcu;
5130 const ip = &zcu.intern_pool;
5131 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5132 const result_ty = cg.typeOfIndex(inst);
5133 const len = @as(usize, @intCast(result_ty.arrayLen(zcu)));
5134 const elements: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..len]);
51355252
5136 const result: WValue = result_value: {5253 try cg.endBlock();
5137 switch (result_ty.zigTypeTag(zcu)) {5254 if (has_else_body) {
5138 .array => {5255 const else_body = cases_it.elseBody();
5139 const result = try cg.allocStack(result_ty);
5140 const elem_ty = result_ty.childType(zcu);
5141 const elem_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
5142 const sentinel = result_ty.sentinel(zcu);
51435256
5144 // When the element type is by reference, we must copy the entire5257 cg.branches.appendAssumeCapacity(.{});
5145 // value. It is therefore safer to move the offset pointer and store5258 const else_deaths = liveness.deaths.len - 1;
5146 // each value individually, instead of using store offsets.5259 try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.deaths[else_deaths].len);
5147 if (isByRef(elem_ty, zcu, cg.target)) {5260 defer {
5148 // copy stack pointer into a temporary local, which is5261 var else_branch = cg.branches.pop().?;
5149 // moved for each element to store each value in the right position.5262 else_branch.deinit(cg.gpa);
5150 const offset = try cg.buildPointerOffset(result, 0, .new);5263 }
5151 for (elements, 0..) |elem, elem_index| {5264 try cg.genBody(else_body);
5152 const elem_val = try cg.resolveInst(elem);5265 } else {
5153 try cg.store(offset, elem_val, elem_ty, 0);5266 try cg.addTag(.@"unreachable");
5267 }
51545268
5155 if (elem_index < elements.len - 1 or sentinel != null) {5269 try cg.endBlock(); // whole switch block end
5156 _ = try cg.buildPointerOffset(offset, elem_size, .modify);
5157 }
5158 }
5159 if (sentinel) |s| {
5160 const val = try cg.resolveValue(s);
5161 try cg.store(offset, val, elem_ty, 0);
5162 }
5163 } else {
5164 var offset: u32 = 0;
5165 for (elements) |elem| {
5166 const elem_val = try cg.resolveInst(elem);
5167 try cg.store(result, elem_val, elem_ty, offset);
5168 offset += elem_size;
5169 }
5170 if (sentinel) |s| {
5171 const val = try cg.resolveValue(s);
5172 try cg.store(result, val, elem_ty, offset);
5173 }
5174 }
5175 break :result_value result;
5176 },
5177 .@"struct" => switch (result_ty.containerLayout(zcu)) {
5178 .@"packed" => {
5179 if (isByRef(result_ty, zcu, cg.target)) {
5180 return cg.fail("TODO: airAggregateInit for packed structs larger than 64 bits", .{});
5181 }
5182 const packed_struct = zcu.typeToPackedStruct(result_ty).?;
5183 const field_types = packed_struct.field_types;
5184 const backing_type = Type.fromInterned(packed_struct.packed_backing_int_type);
5185
5186 // ensure the result is zero'd
5187 const result = try cg.allocLocal(backing_type);
5188 if (backing_type.bitSize(zcu) <= 32)
5189 try cg.addImm32(0)
5190 else
5191 try cg.addImm64(0);
5192 try cg.addLocal(.local_set, result.local.value);
5193
5194 var current_bit: u16 = 0;
5195 for (elements, 0..) |elem, elem_index| {
5196 const field_ty = Type.fromInterned(field_types.get(ip)[elem_index]);
5197 if (!field_ty.hasRuntimeBits(zcu)) continue;
51985270
5199 const shift_val: WValue = if (backing_type.bitSize(zcu) <= 32)5271 if (is_dispatch_loop) {
5200 .{ .imm32 = current_bit }5272 try cg.endBlock(); // dispatch loop end
5201 else5273 }
5202 .{ .imm64 = current_bit };
52035274
5204 const value = try cg.resolveInst(elem);5275 return cg.finishAir(inst, .none, &.{});
5205 const value_bit_size: u16 = @intCast(field_ty.bitSize(zcu));5276}
5206 const int_ty = try pt.intType(.unsigned, value_bit_size);
5207
5208 // load our current result on stack so we can perform all transformations
5209 // using only stack values. Saving the cost of loads and stores.
5210 try cg.emitWValue(result);
5211 const bitcasted = try cg.bitcast(int_ty, field_ty, value);
5212 const extended_val = try cg.intcast(bitcasted, int_ty, backing_type);
5213 // no need to shift any values when the current offset is 0
5214 const shifted = if (current_bit != 0) shifted: {
5215 break :shifted try cg.binOp(extended_val, shift_val, backing_type, .shl);
5216 } else extended_val;
5217 // we ignore the result as we keep it on the stack to assign it directly to `result`
5218 _ = try cg.binOp(.stack, shifted, backing_type, .@"or");
5219 try cg.addLocal(.local_set, result.local.value);
5220 current_bit += value_bit_size;
5221 }
5222 break :result_value result;
5223 },
5224 else => {
5225 const result = try cg.allocStack(result_ty);
5226 const offset = try cg.buildPointerOffset(result, 0, .new); // pointer to offset
5227 var prev_field_offset: u64 = 0;
5228 for (elements, 0..) |elem, elem_index| {
5229 if (try result_ty.structFieldValueComptime(pt, elem_index) != null) continue;
52305277
5231 const elem_ty = result_ty.fieldType(elem_index, zcu);5278fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5232 const field_offset = result_ty.structFieldOffset(elem_index, zcu);5279 const br = cg.air.instructions.items(.data)[@intFromEnum(inst)].br;
5233 _ = try cg.buildPointerOffset(offset, @intCast(field_offset - prev_field_offset), .modify);5280 const switch_loop = cg.blocks.get(br.block_inst).?;
5234 prev_field_offset = field_offset;
52355281
5236 const value = try cg.resolveInst(elem);5282 const operand = try cg.resolveInst(br.operand);
5237 try cg.store(offset, value, elem_ty, 0);5283 try cg.lowerToStack(operand);
5238 }5284 try cg.addLocal(.local_set, switch_loop.value.local.value);
52395285
5240 break :result_value result;5286 const idx: u32 = cg.block_depth - switch_loop.label;
5241 },5287 try cg.addLabel(.br, idx);
5242 },
5243 .vector => return cg.fail("TODO: Wasm backend: implement airAggregateInit for vectors", .{}),
5244 else => unreachable,
5245 }
5246 };
52475288
5248 if (elements.len <= Air.Liveness.bpi - 1) {5289 return cg.finishAir(inst, .none, &.{br.operand});
5249 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
5250 @memcpy(buf[0..elements.len], elements);
5251 return cg.finishAir(inst, result, &buf);
5252 }
5253 var bt = try cg.iterateBigTomb(inst, elements.len);
5254 for (elements) |arg| bt.feed(arg);
5255 return bt.finishAir(result);
5256}5290}
52575291
5258fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5292fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind: enum { value, ptr }) InnerError!void {
5259 const pt = cg.pt;5293 const zcu = cg.pt.zcu;
5260 const zcu = pt.zcu;5294 const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5261 const ip = &zcu.intern_pool;5295 const operand = try cg.resolveInst(un_op);
5262 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;5296 const err_union_ty = switch (op_kind) {
5263 const extra = cg.air.extraData(Air.UnionInit, ty_pl.payload).data;5297 .value => cg.typeOf(un_op),
52645298 .ptr => cg.typeOf(un_op).childType(zcu),
5265 const result = result: {5299 };
5266 const union_ty = cg.typeOfIndex(inst);5300 const pl_ty = err_union_ty.errorUnionPayload(zcu);
5267 const layout = union_ty.unionGetLayout(zcu);
5268 const union_obj = zcu.typeToUnion(union_ty).?;
5269 const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
5270 const field_name = ip.loadEnumType(union_obj.enum_tag_type).field_names.get(ip)[extra.field_index];
52715301
5272 const tag_int = blk: {5302 const result: WValue = result: {
5273 const tag_ty = union_ty.unionTagTypeHypothetical(zcu);5303 if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
5274 const enum_field_index = tag_ty.enumFieldIndex(field_name, zcu).?;5304 switch (opcode) {
5275 const tag_val = try pt.enumValueFieldIndex(tag_ty, enum_field_index);5305 .i32_ne => break :result .{ .imm32 = 0 },
5276 break :blk try cg.lowerConstant(tag_val);5306 .i32_eq => break :result .{ .imm32 = 1 },
5277 };5307 else => unreachable,
5278 if (layout.payload_size == 0) {
5279 if (layout.tag_size == 0) {
5280 break :result .none;
5281 }5308 }
5282 assert(!isByRef(union_ty, zcu, cg.target));
5283 break :result tag_int;
5284 }5309 }
52855310
5286 if (isByRef(union_ty, zcu, cg.target)) {5311 try cg.emitWValue(operand);
5287 const result_ptr = try cg.allocStack(union_ty);5312 if (op_kind == .ptr or pl_ty.hasRuntimeBits(zcu)) {
5288 const payload = try cg.resolveInst(extra.init);5313 try cg.addMemArg(.i32_load16_u, .{
5289 if (layout.tag_align.compare(.gte, layout.payload_align)) {5314 .offset = operand.offset() + @as(u32, @intCast(errUnionErrorOffset(pl_ty, zcu))),
5290 if (isByRef(field_ty, zcu, cg.target)) {5315 .alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),
5291 const payload_ptr = try cg.buildPointerOffset(result_ptr, layout.tag_size, .new);5316 });
5292 try cg.store(payload_ptr, payload, field_ty, 0);5317 }
5293 } else {
5294 try cg.store(result_ptr, payload, field_ty, @intCast(layout.tag_size));
5295 }
52965318
5297 if (layout.tag_size > 0) {5319 // Compare the error value with '0'
5298 try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), 0);5320 try cg.addImm32(0);
5299 }5321 try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode));
5322 break :result .stack;
5323 };
5324 return cg.finishAir(inst, result, &.{un_op});
5325}
5326
5327/// E!T -> T op_is_ptr == false
5328/// *(E!T) -> *T op_is_prt == true
5329fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!void {
5330 const zcu = cg.pt.zcu;
5331 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5332
5333 const operand = try cg.resolveInst(ty_op.operand);
5334 const op_ty = cg.typeOf(ty_op.operand);
5335 const eu_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty;
5336 const payload_ty = eu_ty.errorUnionPayload(zcu);
5337
5338 const result: WValue = result: {
5339 if (!payload_ty.hasRuntimeBits(zcu)) {
5340 if (op_is_ptr) {
5341 break :result cg.reuseOperand(ty_op.operand, operand);
5300 } else {5342 } else {
5301 try cg.store(result_ptr, payload, field_ty, 0);5343 break :result .none;
5302 if (layout.tag_size > 0) {
5303 try cg.store(
5304 result_ptr,
5305 tag_int,
5306 .fromInterned(union_obj.enum_tag_type),
5307 @intCast(layout.payload_size),
5308 );
5309 }
5310 }5344 }
5311 break :result result_ptr;5345 }
5346
5347 const pl_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu));
5348 if (op_is_ptr or isByRef(payload_ty, zcu, cg.target)) {
5349 break :result try cg.buildPointerOffset(operand, pl_offset, .new);
5312 } else {5350 } else {
5313 const operand = try cg.resolveInst(extra.init);5351 assert(isByRef(eu_ty, zcu, cg.target));
5314 const union_int_type = try pt.intType(.unsigned, @as(u16, @intCast(union_ty.bitSize(zcu))));5352 break :result try cg.load(operand, payload_ty, pl_offset);
5315 if (field_ty.zigTypeTag(zcu) == .float) {
5316 const int_type = try pt.intType(.unsigned, @intCast(field_ty.bitSize(zcu)));
5317 const bitcasted = try cg.bitcast(field_ty, int_type, operand);
5318 break :result try cg.trunc(bitcasted, int_type, union_int_type);
5319 } else if (field_ty.isPtrAtRuntime(zcu)) {
5320 const int_type = try pt.intType(.unsigned, @intCast(field_ty.bitSize(zcu)));
5321 break :result try cg.intcast(operand, int_type, union_int_type);
5322 }
5323 break :result try cg.intcast(operand, field_ty, union_int_type);
5324 }5353 }
5325 };5354 };
53265355 return cg.finishAir(inst, result, &.{ty_op.operand});
5327 return cg.finishAir(inst, result, &.{extra.init});
5328}5356}
53295357
5330fn airPrefetch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5358/// E!T -> E op_is_ptr == false
5331 const prefetch = cg.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;5359/// *(E!T) -> E op_is_ptr == true
5332 return cg.finishAir(inst, .none, &.{prefetch.ptr});5360/// NOTE: op_is_ptr will not change return type
5333}5361fn airUnwrapErrUnionError(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!void {
5362 const zcu = cg.pt.zcu;
5363 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
53345364
5335fn airWasmMemorySize(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5365 const operand = try cg.resolveInst(ty_op.operand);
5336 const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;5366 const op_ty = cg.typeOf(ty_op.operand);
5367 const eu_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty;
5368 const payload_ty = eu_ty.errorUnionPayload(zcu);
53375369
5338 try cg.addLabel(.memory_size, pl_op.payload);5370 const result: WValue = result: {
5339 return cg.finishAir(inst, .stack, &.{pl_op.operand});5371 if (eu_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
5372 break :result .{ .imm32 = 0 };
5373 }
5374
5375 const err_offset: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu));
5376 if (op_is_ptr or isByRef(eu_ty, zcu, cg.target)) {
5377 break :result try cg.load(operand, Type.anyerror, err_offset);
5378 } else {
5379 assert(!payload_ty.hasRuntimeBits(zcu));
5380 break :result cg.reuseOperand(ty_op.operand, operand);
5381 }
5382 };
5383 return cg.finishAir(inst, result, &.{ty_op.operand});
5340}5384}
53415385
5342fn airWasmMemoryGrow(cg: *CodeGen, inst: Air.Inst.Index) !void {5386fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5343 const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;5387 const zcu = cg.pt.zcu;
5388 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
53445389
5345 const operand = try cg.resolveInst(pl_op.operand);5390 const operand = try cg.resolveInst(ty_op.operand);
5346 try cg.emitWValue(operand);5391 const err_ty = cg.typeOfIndex(inst);
5347 try cg.addLabel(.memory_grow, pl_op.payload);5392
5348 return cg.finishAir(inst, .stack, &.{pl_op.operand});5393 const pl_ty = cg.typeOf(ty_op.operand);
5394 const result = result: {
5395 if (!pl_ty.hasRuntimeBits(zcu)) {
5396 break :result cg.reuseOperand(ty_op.operand, operand);
5397 }
5398
5399 const err_union = try cg.allocStack(err_ty);
5400 const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);
5401 try cg.store(payload_ptr, operand, pl_ty, 0);
5402
5403 // ensure we also write '0' to the error part, so any present stack value gets overwritten by it.
5404 try cg.emitWValue(err_union);
5405 try cg.addImm32(0);
5406 const err_val_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
5407 try cg.addMemArg(.i32_store16, .{
5408 .offset = err_union.offset() + err_val_offset,
5409 .alignment = 2,
5410 });
5411 break :result err_union;
5412 };
5413 return cg.finishAir(inst, result, &.{ty_op.operand});
5349}5414}
53505415
5351fn cmpOptionals(cg: *CodeGen, lhs: WValue, rhs: WValue, operand_ty: Type, op: std.math.CompareOperator) InnerError!WValue {5416fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5352 const zcu = cg.pt.zcu;5417 const zcu = cg.pt.zcu;
5353 assert(operand_ty.hasRuntimeBits(zcu));5418 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5354 assert(op == .eq or op == .neq);
5355 const payload_ty = operand_ty.optionalChild(zcu);
5356 assert(!isByRef(payload_ty, zcu, cg.target));
5357
5358 var result = try cg.allocLocal(Type.i32);
5359 defer result.free(cg);
53605419
5361 var lhs_null = try cg.allocLocal(Type.i32);5420 const operand = try cg.resolveInst(ty_op.operand);
5362 defer lhs_null.free(cg);5421 const err_ty = ty_op.ty.toType();
5422 const pl_ty = err_ty.errorUnionPayload(zcu);
53635423
5364 try cg.startBlock(.block, .empty);5424 const result = result: {
5425 if (!pl_ty.hasRuntimeBits(zcu)) {
5426 break :result cg.reuseOperand(ty_op.operand, operand);
5427 }
53655428
5366 try cg.addImm32(if (op == .eq) 0 else 1);5429 const err_union = try cg.allocStack(err_ty);
5367 try cg.addLocal(.local_set, result.local.value);5430 // store error value
5431 try cg.store(err_union, operand, Type.anyerror, @intCast(errUnionErrorOffset(pl_ty, zcu)));
53685432
5369 _ = try cg.isNull(lhs, operand_ty, .i32_eq);5433 // write 'undefined' to the payload
5370 try cg.addLocal(.local_tee, lhs_null.local.value);5434 const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);
5371 _ = try cg.isNull(rhs, operand_ty, .i32_eq);5435 const len = @as(u32, @intCast(err_ty.errorUnionPayload(zcu).abiSize(zcu)));
5372 try cg.addTag(.i32_ne);5436 try cg.memset(Type.u8, payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaa });
5373 try cg.addLabel(.br_if, 0); // only one is null
53745437
5375 try cg.addImm32(if (op == .eq) 1 else 0);5438 break :result err_union;
5376 try cg.addLocal(.local_set, result.local.value);5439 };
5440 return cg.finishAir(inst, result, &.{ty_op.operand});
5441}
53775442
5378 try cg.addLocal(.local_get, lhs_null.local.value);5443fn airIsNull(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind: enum { value, ptr }) InnerError!void {
5379 try cg.addLabel(.br_if, 0); // both are null5444 const zcu = cg.pt.zcu;
5445 const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5446 const operand = try cg.resolveInst(un_op);
53805447
5381 _ = try cg.load(lhs, payload_ty, 0);5448 const op_ty = cg.typeOf(un_op);
5382 _ = try cg.load(rhs, payload_ty, 0);5449 const optional_ty = if (op_kind == .ptr) op_ty.childType(zcu) else op_ty;
5383 _ = try cg.cmp(.stack, .stack, payload_ty, op);5450 const result = try cg.isNull(operand, optional_ty, opcode);
5384 try cg.addLocal(.local_set, result.local.value);5451 return cg.finishAir(inst, result, &.{un_op});
5452}
53855453
5386 try cg.endBlock();5454/// For a given type and operand, checks if it's considered `null`.
5455/// NOTE: Leaves the result on the stack
5456fn isNull(cg: *CodeGen, operand: WValue, optional_ty: Type, opcode: std.wasm.Opcode) InnerError!WValue {
5457 const pt = cg.pt;
5458 const zcu = pt.zcu;
5459 try cg.emitWValue(operand);
5460 const payload_ty = optional_ty.optionalChild(zcu);
5461 if (!optional_ty.optionalReprIsPayload(zcu)) {
5462 // When payload is zero-bits, we can treat operand as a value, rather than
5463 // a pointer to the stack value
5464 if (payload_ty.hasRuntimeBits(zcu)) {
5465 const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
5466 return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)});
5467 };
5468 try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = 1 });
5469 }
5470 } else if (payload_ty.isSlice(zcu)) {
5471 switch (cg.ptr_size) {
5472 .wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = 4 }),
5473 .wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = 8 }),
5474 }
5475 }
53875476
5388 try cg.addLocal(.local_get, result.local.value);5477 // Compare the null value with '0'
5478 try cg.addImm32(0);
5479 try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode));
53895480
5390 return .stack;5481 return .stack;
5391}5482}
53925483
5393/// Compares big integers by checking both its high bits and low bits.5484fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5394/// NOTE: Leaves the result of the comparison on top of the stack.
5395/// TODO: Lower this to compiler_rt call when bitsize > 128
5396fn cmpBigInt(cg: *CodeGen, lhs: WValue, rhs: WValue, operand_ty: Type, op: std.math.CompareOperator) InnerError!WValue {
5397 const zcu = cg.pt.zcu;5485 const zcu = cg.pt.zcu;
5398 assert(operand_ty.abiSize(zcu) >= 16);5486 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5399 assert(!(lhs != .stack and rhs == .stack));5487 const opt_ty = cg.typeOf(ty_op.operand);
5400 if (operand_ty.bitSize(zcu) > 128) {5488 const payload_ty = cg.typeOfIndex(inst);
5401 return cg.fail("TODO: Support cmpBigInt for integer bitsize: '{d}'", .{operand_ty.bitSize(zcu)});5489 if (!payload_ty.hasRuntimeBits(zcu)) {
5490 return cg.finishAir(inst, .none, &.{ty_op.operand});
5402 }5491 }
54035492
5404 var lhs_msb = try (try cg.load(lhs, Type.u64, 8)).toLocal(cg, Type.u64);5493 const result = result: {
5405 defer lhs_msb.free(cg);5494 const operand = try cg.resolveInst(ty_op.operand);
5406 var rhs_msb = try (try cg.load(rhs, Type.u64, 8)).toLocal(cg, Type.u64);5495 if (opt_ty.optionalReprIsPayload(zcu)) break :result cg.reuseOperand(ty_op.operand, operand);
5407 defer rhs_msb.free(cg);5496
5497 if (isByRef(payload_ty, zcu, cg.target)) {
5498 break :result try cg.buildPointerOffset(operand, 0, .new);
5499 }
54085500
5409 switch (op) {5501 break :result try cg.load(operand, payload_ty, 0);
5410 .eq, .neq => {5502 };
5411 const xor_high = try cg.binOp(lhs_msb, rhs_msb, Type.u64, .xor);5503 return cg.finishAir(inst, result, &.{ty_op.operand});
5412 const lhs_lsb = try cg.load(lhs, Type.u64, 0);5504}
5413 const rhs_lsb = try cg.load(rhs, Type.u64, 0);
5414 const xor_low = try cg.binOp(lhs_lsb, rhs_lsb, Type.u64, .xor);
5415 const or_result = try cg.binOp(xor_high, xor_low, Type.u64, .@"or");
54165505
5417 switch (op) {5506fn airOptionalPayloadPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5418 .eq => return cg.cmp(or_result, .{ .imm64 = 0 }, Type.u64, .eq),5507 const zcu = cg.pt.zcu;
5419 .neq => return cg.cmp(or_result, .{ .imm64 = 0 }, Type.u64, .neq),5508 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5420 else => unreachable,5509 const operand = try cg.resolveInst(ty_op.operand);
5421 }5510 const opt_ty = cg.typeOf(ty_op.operand).childType(zcu);
5422 },
5423 else => {
5424 const ty = if (operand_ty.isSignedInt(zcu)) Type.i64 else Type.u64;
5425 // leave those value on top of the stack for '.select'
5426 const lhs_lsb = try cg.load(lhs, Type.u64, 0);
5427 const rhs_lsb = try cg.load(rhs, Type.u64, 0);
5428 _ = try cg.cmp(lhs_lsb, rhs_lsb, Type.u64, op);
5429 _ = try cg.cmp(lhs_msb, rhs_msb, ty, op);
5430 _ = try cg.cmp(lhs_msb, rhs_msb, ty, .eq);
5431 try cg.addTag(.select);
5432 },
5433 }
54345511
5435 return .stack;5512 const result = result: {
5513 const payload_ty = opt_ty.optionalChild(zcu);
5514 if (!payload_ty.hasRuntimeBits(zcu) or opt_ty.optionalReprIsPayload(zcu)) {
5515 break :result cg.reuseOperand(ty_op.operand, operand);
5516 }
5517
5518 break :result try cg.buildPointerOffset(operand, 0, .new);
5519 };
5520 return cg.finishAir(inst, result, &.{ty_op.operand});
5436}5521}
54375522
5438fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5523fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5439 const pt = cg.pt;5524 const pt = cg.pt;
5440 const zcu = pt.zcu;5525 const zcu = pt.zcu;
5441 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;5526 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5442 const un_ty = cg.typeOf(bin_op.lhs).childType(zcu);5527 const operand = try cg.resolveInst(ty_op.operand);
5443 const tag_ty = cg.typeOf(bin_op.rhs);5528 const opt_ty = cg.typeOf(ty_op.operand).childType(zcu);
5444 const layout = un_ty.unionGetLayout(zcu);5529 const payload_ty = opt_ty.optionalChild(zcu);
5445 if (layout.tag_size == 0) return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
54465530
5447 const union_ptr = try cg.resolveInst(bin_op.lhs);5531 if (opt_ty.optionalReprIsPayload(zcu)) {
5448 const new_tag = try cg.resolveInst(bin_op.rhs);5532 return cg.finishAir(inst, operand, &.{ty_op.operand});
5449 if (layout.payload_size == 0) {
5450 try cg.store(union_ptr, new_tag, tag_ty, 0);
5451 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
5452 }5533 }
54535534
5454 // when the tag alignment is smaller than the payload, the field will be stored5535 const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
5455 // after the payload.5536 return cg.fail("Optional type {f} too big to fit into stack frame", .{opt_ty.fmt(pt)});
5456 const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align)) blk: {5537 };
5457 break :blk @intCast(layout.payload_size);5538
5458 } else 0;5539 try cg.emitWValue(operand);
5459 try cg.store(union_ptr, new_tag, tag_ty, offset);5540 try cg.addImm32(1);
5460 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });5541 try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = 1 });
5542
5543 const result = try cg.buildPointerOffset(operand, 0, .new);
5544 return cg.finishAir(inst, result, &.{ty_op.operand});
5461}5545}
54625546
5463fn airGetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5547fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5464 const zcu = cg.pt.zcu;
5465 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5548 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5549 const payload_ty = cg.typeOf(ty_op.operand);
5550 const pt = cg.pt;
5551 const zcu = pt.zcu;
5552
5553 const result = result: {
5554 if (!payload_ty.hasRuntimeBits(zcu)) {
5555 const non_null_bit = try cg.allocStack(Type.u1);
5556 try cg.emitWValue(non_null_bit);
5557 try cg.addImm32(1);
5558 try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = 1 });
5559 break :result non_null_bit;
5560 }
5561
5562 const operand = try cg.resolveInst(ty_op.operand);
5563 const op_ty = cg.typeOfIndex(inst);
5564 if (op_ty.optionalReprIsPayload(zcu)) {
5565 break :result cg.reuseOperand(ty_op.operand, operand);
5566 }
5567 const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
5568 return cg.fail("Optional type {f} too big to fit into stack frame", .{op_ty.fmt(pt)});
5569 };
5570
5571 // Create optional type, set the non-null bit, and store the operand inside the optional type
5572 const result_ptr = try cg.allocStack(op_ty);
5573 try cg.emitWValue(result_ptr);
5574 try cg.addImm32(1);
5575 try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = 1 });
54665576
5467 const un_ty = cg.typeOf(ty_op.operand);5577 const payload_ptr = try cg.buildPointerOffset(result_ptr, 0, .new);
5468 const tag_ty = cg.typeOfIndex(inst);5578 try cg.store(payload_ptr, operand, payload_ty, 0);
5469 const layout = un_ty.unionGetLayout(zcu);5579 break :result result_ptr;
5470 if (layout.tag_size == 0) return cg.finishAir(inst, .none, &.{ty_op.operand});5580 };
54715581
5472 const operand = try cg.resolveInst(ty_op.operand);
5473 // when the tag alignment is smaller than the payload, the field will be stored
5474 // after the payload.
5475 const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align))
5476 @intCast(layout.payload_size)
5477 else
5478 0;
5479 const result = try cg.load(operand, tag_ty, offset);
5480 return cg.finishAir(inst, result, &.{ty_op.operand});5582 return cg.finishAir(inst, result, &.{ty_op.operand});
5481}5583}
54825584
5483fn airFpext(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5585fn airSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5484 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5586 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5587 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
54855588
5486 const dest_ty = cg.typeOfIndex(inst);5589 const lhs = try cg.resolveInst(bin_op.lhs);
5487 const operand = try cg.resolveInst(ty_op.operand);5590 const rhs = try cg.resolveInst(bin_op.rhs);
5488 const result = try cg.fpext(operand, cg.typeOf(ty_op.operand), dest_ty);5591 const slice_ty = cg.typeOfIndex(inst);
5489 return cg.finishAir(inst, result, &.{ty_op.operand});
5490}
54915592
5492/// Extends a float from a given `Type` to a larger wanted `Type`, leaving the5593 const slice = try cg.allocStack(slice_ty);
5493/// result on the stack.5594 try cg.store(slice, lhs, Type.usize, 0);
5494fn fpext(cg: *CodeGen, operand: WValue, given: Type, wanted: Type) InnerError!WValue {5595 try cg.store(slice, rhs, Type.usize, cg.ptrSize());
5495 const given_bits = given.floatBits(cg.target);
5496 const wanted_bits = wanted.floatBits(cg.target);
54975596
5498 const intrinsic: Mir.Intrinsic = switch (given_bits) {5597 return cg.finishAir(inst, slice, &.{ bin_op.lhs, bin_op.rhs });
5499 16 => switch (wanted_bits) {
5500 32 => {
5501 assert(.stack == try cg.callIntrinsic(.__extendhfsf2, &.{.f16_type}, Type.f32, &.{operand}));
5502 return .stack;
5503 },
5504 64 => {
5505 assert(.stack == try cg.callIntrinsic(.__extendhfsf2, &.{.f16_type}, Type.f32, &.{operand}));
5506 try cg.addTag(.f64_promote_f32);
5507 return .stack;
5508 },
5509 80 => .__extendhfxf2,
5510 128 => .__extendhftf2,
5511 else => unreachable,
5512 },
5513 32 => switch (wanted_bits) {
5514 64 => {
5515 try cg.emitWValue(operand);
5516 try cg.addTag(.f64_promote_f32);
5517 return .stack;
5518 },
5519 80 => .__extendsfxf2,
5520 128 => .__extendsftf2,
5521 else => unreachable,
5522 },
5523 64 => switch (wanted_bits) {
5524 80 => .__extenddfxf2,
5525 128 => .__extenddftf2,
5526 else => unreachable,
5527 },
5528 80 => switch (wanted_bits) {
5529 128 => .__extendxftf2,
5530 else => unreachable,
5531 },
5532 else => unreachable,
5533 };
5534 return cg.callIntrinsic(intrinsic, &.{given.ip_index}, wanted, &.{operand});
5535}5598}
55365599
5537fn airFptrunc(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5600fn airSliceLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5538 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5601 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
55395602
5540 const dest_ty = cg.typeOfIndex(inst);
5541 const operand = try cg.resolveInst(ty_op.operand);5603 const operand = try cg.resolveInst(ty_op.operand);
5542 const result = try cg.fptrunc(operand, cg.typeOf(ty_op.operand), dest_ty);5604 return cg.finishAir(inst, try cg.sliceLen(operand), &.{ty_op.operand});
5543 return cg.finishAir(inst, result, &.{ty_op.operand});
5544}
5545
5546/// Truncates a float from a given `Type` to its wanted `Type`, leaving the
5547/// result on the stack.
5548fn fptrunc(cg: *CodeGen, operand: WValue, given: Type, wanted: Type) InnerError!WValue {
5549 const given_bits = given.floatBits(cg.target);
5550 const wanted_bits = wanted.floatBits(cg.target);
5551
5552 const intrinsic: Mir.Intrinsic = switch (given_bits) {
5553 32 => switch (wanted_bits) {
5554 16 => {
5555 return cg.callIntrinsic(.__truncsfhf2, &.{.f32_type}, Type.f16, &.{operand});
5556 },
5557 else => unreachable,
5558 },
5559 64 => switch (wanted_bits) {
5560 16 => {
5561 try cg.emitWValue(operand);
5562 try cg.addTag(.f32_demote_f64);
5563 return cg.callIntrinsic(.__truncsfhf2, &.{.f32_type}, Type.f16, &.{.stack});
5564 },
5565 32 => {
5566 try cg.emitWValue(operand);
5567 try cg.addTag(.f32_demote_f64);
5568 return .stack;
5569 },
5570 else => unreachable,
5571 },
5572 80 => switch (wanted_bits) {
5573 16 => .__truncxfhf2,
5574 32 => .__truncxfsf2,
5575 64 => .__truncxfdf2,
5576 else => unreachable,
5577 },
5578 128 => switch (wanted_bits) {
5579 16 => .__trunctfhf2,
5580 32 => .__trunctfsf2,
5581 64 => .__trunctfdf2,
5582 80 => .__trunctfxf2,
5583 else => unreachable,
5584 },
5585 else => unreachable,
5586 };
5587 return cg.callIntrinsic(intrinsic, &.{given.ip_index}, wanted, &.{operand});
5588}5605}
55895606
5590fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5607fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5591 const zcu = cg.pt.zcu;5608 const zcu = cg.pt.zcu;
5592 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5609 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
55935610
5594 const err_set_ty = cg.typeOf(ty_op.operand).childType(zcu);5611 const slice_ty = cg.typeOf(bin_op.lhs);
5595 const payload_ty = err_set_ty.errorUnionPayload(zcu);5612 const slice = try cg.resolveInst(bin_op.lhs);
5596 const operand = try cg.resolveInst(ty_op.operand);5613 const index = try cg.resolveInst(bin_op.rhs);
5614 const elem_ty = slice_ty.childType(zcu);
5615 const elem_size = elem_ty.abiSize(zcu);
55975616
5598 // set error-tag to '0' to annotate error union is non-error5617 // load pointer onto stack
5599 try cg.store(5618 _ = try cg.load(slice, Type.usize, 0);
5600 operand,
5601 .{ .imm32 = 0 },
5602 Type.anyerror,
5603 @intCast(errUnionErrorOffset(payload_ty, zcu)),
5604 );
56055619
5606 const result = result: {5620 // calculate index into slice
5607 if (!payload_ty.hasRuntimeBits(zcu)) {5621 try cg.emitWValue(index);
5608 break :result cg.reuseOperand(ty_op.operand, operand);5622 try cg.addImm32(@intCast(elem_size));
5609 }5623 try cg.addTag(.i32_mul);
5624 try cg.addTag(.i32_add);
56105625
5611 break :result try cg.buildPointerOffset(operand, @as(u32, @intCast(errUnionPayloadOffset(payload_ty, zcu))), .new);5626 const elem_result = if (isByRef(elem_ty, zcu, cg.target))
5612 };5627 .stack
5613 return cg.finishAir(inst, result, &.{ty_op.operand});5628 else
5629 try cg.load(.stack, elem_ty, 0);
5630
5631 return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
5614}5632}
56155633
5616fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5634fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5617 const pt = cg.pt;5635 const zcu = cg.pt.zcu;
5618 const zcu = pt.zcu;
5619 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;5636 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5620 const extra = cg.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;5637 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
56215638
5622 const field_ptr = try cg.resolveInst(extra.field_ptr);5639 const elem_ty = ty_pl.ty.toType().childType(zcu);
5623 const parent_ptr_ty = cg.typeOfIndex(inst);5640 const elem_size = elem_ty.abiSize(zcu);
5624 const parent_ty = parent_ptr_ty.childType(zcu);
5625 const field_ptr_ty = cg.typeOf(extra.field_ptr);
5626 const field_index = extra.field_index;
5627 const field_offset = switch (parent_ty.containerLayout(zcu)) {
5628 .auto, .@"extern" => parent_ty.structFieldOffset(field_index, zcu),
5629 .@"packed" => offset: {
5630 const parent_ptr_offset = parent_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset;
5631 const field_offset = if (zcu.typeToStruct(parent_ty)) |loaded_struct| zcu.structPackedFieldBitOffset(loaded_struct, field_index) else 0;
5632 const field_ptr_offset = field_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset;
5633 break :offset @divExact(parent_ptr_offset + field_offset - field_ptr_offset, 8);
5634 },
5635 };
56365641
5637 const result = if (field_offset != 0) result: {5642 const slice = try cg.resolveInst(bin_op.lhs);
5638 const base = try cg.buildPointerOffset(field_ptr, 0, .new);5643 const index = try cg.resolveInst(bin_op.rhs);
5639 try cg.addLocal(.local_get, base.local.value);
5640 try cg.addImm32(@intCast(field_offset));
5641 try cg.addTag(.i32_sub);
5642 try cg.addLocal(.local_set, base.local.value);
5643 break :result base;
5644 } else cg.reuseOperand(extra.field_ptr, field_ptr);
56455644
5646 return cg.finishAir(inst, result, &.{extra.field_ptr});5645 _ = try cg.load(slice, Type.usize, 0);
5647}
56485646
5649fn sliceOrArrayPtr(cg: *CodeGen, ptr: WValue, ptr_ty: Type) InnerError!WValue {5647 // calculate index into slice
5650 const zcu = cg.pt.zcu;5648 try cg.emitWValue(index);
5651 if (ptr_ty.isSlice(zcu)) {5649 try cg.addImm32(@intCast(elem_size));
5652 return cg.slicePtr(ptr);5650 try cg.addTag(.i32_mul);
5653 } else {5651 try cg.addTag(.i32_add);
5654 return ptr;5652
5655 }5653 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
5656}5654}
56575655
5658fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5656fn airSlicePtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5659 const zcu = cg.pt.zcu;5657 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5660 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;5658 const operand = try cg.resolveInst(ty_op.operand);
5661 const dst = try cg.resolveInst(bin_op.lhs);5659 return cg.finishAir(inst, try cg.slicePtr(operand), &.{ty_op.operand});
5662 const dst_ty = cg.typeOf(bin_op.lhs);5660}
5663 const ptr_elem_ty = dst_ty.childType(zcu);
5664 const src = try cg.resolveInst(bin_op.rhs);
5665 const src_ty = cg.typeOf(bin_op.rhs);
5666 const len = switch (dst_ty.ptrSize(zcu)) {
5667 .slice => blk: {
5668 const slice_len = try cg.sliceLen(dst);
5669 if (ptr_elem_ty.abiSize(zcu) != 1) {
5670 try cg.emitWValue(slice_len);
5671 try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) });
5672 try cg.addTag(.i32_mul);
5673 try cg.addLocal(.local_set, slice_len.local.value);
5674 }
5675 break :blk slice_len;
5676 },
5677 .one => @as(WValue, .{
5678 .imm32 = @as(u32, @intCast(ptr_elem_ty.arrayLen(zcu) * ptr_elem_ty.childType(zcu).abiSize(zcu))),
5679 }),
5680 .c, .many => unreachable,
5681 };
5682 const dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty);
5683 const src_ptr = try cg.sliceOrArrayPtr(src, src_ty);
5684 try cg.memcpy(dst_ptr, src_ptr, len);
56855661
5686 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });5662fn slicePtr(cg: *CodeGen, operand: WValue) InnerError!WValue {
5663 const ptr = try cg.load(operand, Type.usize, 0);
5664 return ptr.toLocal(cg, Type.usize);
5687}5665}
56885666
5689fn airRetAddr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5667fn sliceLen(cg: *CodeGen, operand: WValue) InnerError!WValue {
5690 // TODO: Implement this properly once stack serialization is solved5668 const len = try cg.load(operand, Type.usize, cg.ptrSize());
5691 return cg.finishAir(inst, switch (cg.ptr_size) {5669 return len.toLocal(cg, Type.usize);
5692 .wasm32 => .{ .imm32 = 0 },
5693 .wasm64 => .{ .imm64 = 0 },
5694 }, &.{});
5695}5670}
56965671
5697fn airPopcount(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5672fn airArrayToSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5698 const pt = cg.pt;5673 const zcu = cg.pt.zcu;
5699 const zcu = pt.zcu;
5700 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5674 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
57015675
5702 const operand = try cg.resolveInst(ty_op.operand);5676 const operand = try cg.resolveInst(ty_op.operand);
5703 const op_ty = cg.typeOf(ty_op.operand);5677 const array_ty = cg.typeOf(ty_op.operand).childType(zcu);
5678 const slice_ty = ty_op.ty.toType();
5679
5680 // create a slice on the stack
5681 const slice_local = try cg.allocStack(slice_ty);
57045682
5705 if (op_ty.zigTypeTag(zcu) == .vector) {5683 // store the array ptr in the slice
5706 return cg.fail("TODO: Implement @popCount for vectors", .{});5684 if (array_ty.hasRuntimeBits(zcu)) {
5685 try cg.store(slice_local, operand, Type.usize, 0);
5707 }5686 }
57085687
5709 const int_info = op_ty.intInfo(zcu);5688 // store the length of the array in the slice
5710 const bits = int_info.bits;5689 const array_len: u32 = @intCast(array_ty.arrayLen(zcu));
5711 const wasm_bits = toWasmBits(bits) orelse {5690 try cg.store(slice_local, .{ .imm32 = array_len }, Type.usize, cg.ptrSize());
5712 return cg.fail("TODO: Implement @popCount for integers with bitsize '{d}'", .{bits});
5713 };
57145691
5715 switch (wasm_bits) {5692 return cg.finishAir(inst, slice_local, &.{ty_op.operand});
5716 32 => {5693}
5717 try cg.emitWValue(operand);5694
5718 if (op_ty.isSignedInt(zcu) and bits != wasm_bits) {5695fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5719 _ = try cg.wrapOperand(.stack, try pt.intType(.unsigned, bits));5696 const zcu = cg.pt.zcu;
5720 }5697 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
5721 try cg.addTag(.i32_popcnt);5698
5722 },5699 const ptr_ty = cg.typeOf(bin_op.lhs);
5723 64 => {5700 const ptr = try cg.resolveInst(bin_op.lhs);
5724 try cg.emitWValue(operand);5701 const index = try cg.resolveInst(bin_op.rhs);
5725 if (op_ty.isSignedInt(zcu) and bits != wasm_bits) {5702 const elem_ty = ptr_ty.childType(zcu);
5726 _ = try cg.wrapOperand(.stack, try pt.intType(.unsigned, bits));5703 const elem_size = elem_ty.abiSize(zcu);
5727 }5704
5728 try cg.addTag(.i64_popcnt);5705 // load pointer onto the stack
5729 try cg.addTag(.i32_wrap_i64);5706 if (ptr_ty.isSlice(zcu)) {
5730 try cg.emitWValue(operand);5707 _ = try cg.load(ptr, Type.usize, 0);
5731 },5708 } else {
5732 128 => {5709 try cg.lowerToStack(ptr);
5733 _ = try cg.load(operand, Type.u64, 0);
5734 try cg.addTag(.i64_popcnt);
5735 _ = try cg.load(operand, Type.u64, 8);
5736 if (op_ty.isSignedInt(zcu) and bits != wasm_bits) {
5737 _ = try cg.wrapOperand(.stack, try pt.intType(.unsigned, bits - 64));
5738 }
5739 try cg.addTag(.i64_popcnt);
5740 try cg.addTag(.i64_add);
5741 try cg.addTag(.i32_wrap_i64);
5742 },
5743 else => unreachable,
5744 }5710 }
57455711
5746 return cg.finishAir(inst, .stack, &.{ty_op.operand});5712 // calculate index into slice
5713 try cg.emitWValue(index);
5714 try cg.addImm32(@intCast(elem_size));
5715 try cg.addTag(.i32_mul);
5716 try cg.addTag(.i32_add);
5717
5718 const elem_result = if (isByRef(elem_ty, zcu, cg.target))
5719 .stack
5720 else
5721 try cg.load(.stack, elem_ty, 0);
5722
5723 return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
5747}5724}
57485725
5749fn airBitReverse(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5726fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5750 const zcu = cg.pt.zcu;5727 const zcu = cg.pt.zcu;
5751 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5728 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5729 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
57525730
5753 const operand = try cg.resolveInst(ty_op.operand);5731 const ptr_ty = cg.typeOf(bin_op.lhs);
5754 const ty = cg.typeOf(ty_op.operand);5732 const elem_ty = ty_pl.ty.toType().childType(zcu);
5733 const elem_size = elem_ty.abiSize(zcu);
57555734
5756 if (ty.zigTypeTag(zcu) == .vector) {5735 const ptr = try cg.resolveInst(bin_op.lhs);
5757 return cg.fail("TODO: Implement @bitReverse for vectors", .{});5736 const index = try cg.resolveInst(bin_op.rhs);
5758 }
57595737
5760 const int_info = ty.intInfo(zcu);5738 // load pointer onto the stack
5761 const bits = int_info.bits;5739 if (ptr_ty.isSlice(zcu)) {
5762 const wasm_bits = toWasmBits(bits) orelse {5740 _ = try cg.load(ptr, Type.usize, 0);
5763 return cg.fail("TODO: Implement @bitReverse for integers with bitsize '{d}'", .{bits});5741 } else {
5764 };5742 try cg.lowerToStack(ptr);
5743 }
57655744
5766 switch (wasm_bits) {5745 // calculate index into ptr
5767 32 => {5746 try cg.emitWValue(index);
5768 const intrin_ret = try cg.callIntrinsic(5747 try cg.addImm32(@intCast(elem_size));
5769 .__bitreversesi2,5748 try cg.addTag(.i32_mul);
5770 &.{.u32_type},5749 try cg.addTag(.i32_add);
5771 Type.u32,
5772 &.{operand},
5773 );
5774 const result = if (bits == 32)
5775 intrin_ret
5776 else
5777 try cg.binOp(intrin_ret, .{ .imm32 = 32 - bits }, ty, .shr);
5778 return cg.finishAir(inst, result, &.{ty_op.operand});
5779 },
5780 64 => {
5781 const intrin_ret = try cg.callIntrinsic(
5782 .__bitreversedi2,
5783 &.{.u64_type},
5784 Type.u64,
5785 &.{operand},
5786 );
5787 const result = if (bits == 64)
5788 intrin_ret
5789 else
5790 try cg.binOp(intrin_ret, .{ .imm64 = 64 - bits }, ty, .shr);
5791 return cg.finishAir(inst, result, &.{ty_op.operand});
5792 },
5793 128 => {
5794 const result = try cg.allocStack(ty);
57955750
5796 try cg.emitWValue(result);5751 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
5797 const first_half = try cg.load(operand, Type.u64, 8);
5798 const intrin_ret_first = try cg.callIntrinsic(
5799 .__bitreversedi2,
5800 &.{.u64_type},
5801 Type.u64,
5802 &.{first_half},
5803 );
5804 try cg.emitWValue(intrin_ret_first);
5805 if (bits < 128) {
5806 try cg.emitWValue(.{ .imm64 = 128 - bits });
5807 try cg.addTag(.i64_shr_u);
5808 }
5809 try cg.emitWValue(result);
5810 const second_half = try cg.load(operand, Type.u64, 0);
5811 const intrin_ret_second = try cg.callIntrinsic(
5812 .__bitreversedi2,
5813 &.{.u64_type},
5814 Type.u64,
5815 &.{second_half},
5816 );
5817 try cg.emitWValue(intrin_ret_second);
5818 if (bits == 128) {
5819 try cg.store(.stack, .stack, Type.u64, result.offset() + 8);
5820 try cg.store(.stack, .stack, Type.u64, result.offset());
5821 } else {
5822 var tmp = try cg.allocLocal(Type.u64);
5823 defer tmp.free(cg);
5824 try cg.addLocal(.local_tee, tmp.local.value);
5825 try cg.emitWValue(.{ .imm64 = 128 - bits });
5826 if (ty.isSignedInt(zcu)) {
5827 try cg.addTag(.i64_shr_s);
5828 } else {
5829 try cg.addTag(.i64_shr_u);
5830 }
5831 try cg.store(.stack, .stack, Type.u64, result.offset() + 8);
5832 try cg.addLocal(.local_get, tmp.local.value);
5833 try cg.emitWValue(.{ .imm64 = bits - 64 });
5834 try cg.addTag(.i64_shl);
5835 try cg.addTag(.i64_or);
5836 try cg.store(.stack, .stack, Type.u64, result.offset());
5837 }
5838 return cg.finishAir(inst, result, &.{ty_op.operand});
5839 },
5840 else => unreachable,
5841 }
5842}5752}
58435753
5844fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5754fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerError!void {
5845 const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;5755 const zcu = cg.pt.zcu;
5846 const operand = try cg.resolveInst(un_op);5756 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5847 // Each entry to this table is a slice (ptr+len).5757 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
5848 // The operand in this instruction represents the index within this table.5758
5849 // This means to get the final name, we emit the base pointer and then perform5759 const ptr = try cg.resolveInst(bin_op.lhs);
5850 // pointer arithmetic to find the pointer to this slice and return that.5760 const offset = try cg.resolveInst(bin_op.rhs);
5851 //5761 const ptr_ty = cg.typeOf(bin_op.lhs);
5852 // As the names are global and the slice elements are constant, we do not have5762 const pointee_ty = switch (ptr_ty.ptrSize(zcu)) {
5853 // to make a copy of the ptr+value but can point towards them directly.5763 .one => ptr_ty.childType(zcu).childType(zcu), // ptr to array, so get array element type
5854 const pt = cg.pt;5764 else => ptr_ty.childType(zcu),
5855 const name_ty = Type.slice_const_u8_sentinel_0;5765 };
5856 const abi_size = name_ty.abiSize(pt.zcu);5766
5767 try cg.lowerToStack(ptr);
5768 try cg.emitWValue(offset);
58575769
5858 // Lowers to a i32.const or i64.const with the error table memory address.
5859 cg.error_name_table_ref_count += 1;
5860 try cg.addTag(.error_name_table_ref);
5861 try cg.emitWValue(operand);
5862 switch (cg.ptr_size) {5770 switch (cg.ptr_size) {
5863 .wasm32 => {5771 .wasm32 => {
5864 try cg.addImm32(@intCast(abi_size));5772 try cg.addImm32(@intCast(pointee_ty.abiSize(zcu)));
5865 try cg.addTag(.i32_mul);5773 try cg.addTag(.i32_mul);
5866 try cg.addTag(.i32_add);5774 try cg.addTag(switch (op) {
5775 .add => .i32_add,
5776 .sub => .i32_sub,
5777 });
5867 },5778 },
5868 .wasm64 => {5779 .wasm64 => {
5869 try cg.addImm64(abi_size);5780 try cg.addImm64(pointee_ty.abiSize(zcu));
5870 try cg.addTag(.i64_mul);5781 try cg.addTag(.i64_mul);
5871 try cg.addTag(.i64_add);5782 try cg.addTag(switch (op) {
5783 .add => .i64_add,
5784 .sub => .i64_sub,
5785 });
5872 },5786 },
5873 }5787 }
58745788
5875 return cg.finishAir(inst, .stack, &.{un_op});5789 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
5876}
5877
5878fn airPtrSliceFieldPtr(cg: *CodeGen, inst: Air.Inst.Index, offset: u32) InnerError!void {
5879 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5880 const slice_ptr = try cg.resolveInst(ty_op.operand);
5881 const result = try cg.buildPointerOffset(slice_ptr, offset, .new);
5882 return cg.finishAir(inst, result, &.{ty_op.operand});
5883}5790}
58845791
5885/// NOTE: Allocates place for result on virtual stack, when integer size > 64 bits5792fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
5886fn intZeroValue(cg: *CodeGen, ty: Type) InnerError!WValue {
5887 const zcu = cg.pt.zcu;5793 const zcu = cg.pt.zcu;
5888 const int_info = ty.intInfo(zcu);5794 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
5889 const wasm_bits = toWasmBits(int_info.bits) orelse {
5890 return cg.fail("TODO: Implement intZeroValue for integer bitsize: {d}", .{int_info.bits});
5891 };
5892 switch (wasm_bits) {
5893 32 => return .{ .imm32 = 0 },
5894 64 => return .{ .imm64 = 0 },
5895 128 => {
5896 const result = try cg.allocStack(ty);
5897 try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0);
5898 try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8);
5899 return result;
5900 },
5901 else => unreachable,
5902 }
5903}
5904
5905fn airAddSubWithOverflow(cg: *CodeGen, inst: Air.Inst.Index, op: Op) InnerError!void {
5906 assert(op == .add or op == .sub);
5907 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5908 const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data;
5909
5910 const lhs = try cg.resolveInst(extra.lhs);
5911 const rhs = try cg.resolveInst(extra.rhs);
5912 const ty = cg.typeOf(extra.lhs);
5913 const pt = cg.pt;
5914 const zcu = pt.zcu;
5915
5916 if (ty.zigTypeTag(zcu) == .vector) {
5917 return cg.fail("TODO: Implement overflow arithmetic for vectors", .{});
5918 }
5919
5920 const int_info = ty.intInfo(zcu);
5921 const is_signed = int_info.signedness == .signed;
5922 if (int_info.bits > 128) {
5923 return cg.fail("TODO: Implement {{add/sub}}_with_overflow for integer bitsize: {d}", .{int_info.bits});
5924 }
5925
5926 const op_result = try cg.wrapBinOp(lhs, rhs, ty, op);
5927 var op_tmp = try op_result.toLocal(cg, ty);
5928 defer op_tmp.free(cg);
59295795
5930 const cmp_op: std.math.CompareOperator = switch (op) {5796 const ptr = try cg.resolveInst(bin_op.lhs);
5931 .add => .lt,5797 const ptr_ty = cg.typeOf(bin_op.lhs);
5932 .sub => .gt,5798 const value = try cg.resolveInst(bin_op.rhs);
5933 else => unreachable,5799 const len = switch (ptr_ty.ptrSize(zcu)) {
5800 .slice => try cg.sliceLen(ptr),
5801 .one => @as(WValue, .{ .imm32 = @as(u32, @intCast(ptr_ty.childType(zcu).arrayLen(zcu))) }),
5802 .c, .many => unreachable,
5934 };5803 };
5935 const overflow_bit = if (is_signed) blk: {
5936 const zero = try intZeroValue(cg, ty);
5937 const rhs_is_neg = try cg.cmp(rhs, zero, ty, .lt);
5938 const overflow_cmp = try cg.cmp(op_tmp, lhs, ty, cmp_op);
5939 break :blk try cg.cmp(rhs_is_neg, overflow_cmp, Type.u1, .neq);
5940 } else try cg.cmp(op_tmp, lhs, ty, cmp_op);
5941 var bit_tmp = try overflow_bit.toLocal(cg, Type.u1);
5942 defer bit_tmp.free(cg);
59435804
5944 const result = try cg.allocStack(cg.typeOfIndex(inst));5805 const elem_ty = if (ptr_ty.ptrSize(zcu) == .one)
5945 const offset: u32 = @intCast(ty.abiSize(zcu));5806 ptr_ty.childType(zcu).childType(zcu)
5946 try cg.store(result, op_tmp, ty, 0);5807 else
5947 try cg.store(result, bit_tmp, Type.u1, offset);5808 ptr_ty.childType(zcu);
5948
5949 return cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs });
5950}
5951
5952fn airShlWithOverflow(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5953 const pt = cg.pt;
5954 const zcu = pt.zcu;
5955 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5956 const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data;
5957
5958 const lhs = try cg.resolveInst(extra.lhs);
5959 const rhs = try cg.resolveInst(extra.rhs);
5960 const ty = cg.typeOf(extra.lhs);
5961 const rhs_ty = cg.typeOf(extra.rhs);
59625809
5963 if (ty.isVector(zcu)) {5810 if (!safety and bin_op.rhs == .undef) {
5964 if (!rhs_ty.isVector(zcu)) {5811 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
5965 return cg.fail("TODO: implement vector 'shl_with_overflow' with scalar rhs", .{});
5966 } else {
5967 return cg.fail("TODO: implement vector 'shl_with_overflow'", .{});
5968 }
5969 }5812 }
59705813
5971 const int_info = ty.intInfo(zcu);5814 const dst_ptr = try cg.sliceOrArrayPtr(ptr, ptr_ty);
5972 const wasm_bits = toWasmBits(int_info.bits) orelse {5815 try cg.memset(elem_ty, dst_ptr, len, value);
5973 return cg.fail("TODO: implement 'shl_with_overflow' for integer bitsize: {d}", .{int_info.bits});
5974 };
5975
5976 // Ensure rhs is coerced to lhs as they must have the same WebAssembly types
5977 // before we can perform any binary operation.
5978 const rhs_wasm_bits = toWasmBits(rhs_ty.intInfo(zcu).bits).?;
5979 // If wasm_bits == 128, compiler-rt expects i32 for shift
5980 const rhs_final = if (wasm_bits != rhs_wasm_bits and wasm_bits == 64) blk: {
5981 const rhs_casted = try cg.intcast(rhs, rhs_ty, ty);
5982 break :blk try rhs_casted.toLocal(cg, ty);
5983 } else rhs;
5984
5985 var shl = try (try cg.wrapBinOp(lhs, rhs_final, ty, .shl)).toLocal(cg, ty);
5986 defer shl.free(cg);
5987
5988 const overflow_bit = blk: {
5989 const shr = try cg.binOp(shl, rhs_final, ty, .shr);
5990 break :blk try cg.cmp(shr, lhs, ty, .neq);
5991 };
5992 var overflow_local = try overflow_bit.toLocal(cg, Type.u1);
5993 defer overflow_local.free(cg);
5994
5995 const result = try cg.allocStack(cg.typeOfIndex(inst));
5996 const offset: u32 = @intCast(ty.abiSize(zcu));
5997 try cg.store(result, shl, ty, 0);
5998 try cg.store(result, overflow_local, Type.u1, offset);
59995816
6000 return cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs });5817 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
6001}5818}
60025819
6003fn airMulWithOverflow(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5820/// Sets a region of memory at `ptr` to the value of `value`
6004 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;5821/// When the user has enabled the bulk_memory feature, we lower
6005 const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data;5822/// this to wasm's memset instruction. When the feature is not present,
5823/// we implement it manually.
5824fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) InnerError!void {
5825 const zcu = cg.pt.zcu;
5826 const abi_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
60065827
6007 const lhs = try cg.resolveInst(extra.lhs);5828 // When bulk_memory is enabled, we lower it to wasm's memset instruction.
6008 const rhs = try cg.resolveInst(extra.rhs);5829 // If not, we lower it ourselves.
6009 const ty = cg.typeOf(extra.lhs);5830 if (cg.target.cpu.has(.wasm, .bulk_memory) and abi_size == 1) {
6010 const pt = cg.pt;5831 const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0);
6011 const zcu = pt.zcu;
60125832
6013 if (ty.zigTypeTag(zcu) == .vector) {5833 if (!len0_ok) {
6014 return cg.fail("TODO: Implement overflow arithmetic for vectors", .{});5834 try cg.startBlock(.block, .empty);
6015 }
60165835
6017 // We store the bit if it's overflowed or not in this. As it's zero-initialized5836 // Even if `len` is zero, the spec requires an implementation to trap if `ptr + len` is
6018 // we only need to update it if an overflow (or underflow) occurred.5837 // out of memory bounds. This can easily happen in Zig in a case such as:
6019 var overflow_bit = try cg.ensureAllocLocal(Type.u1);5838 //
6020 defer overflow_bit.free(cg);5839 // const ptr: [*]u8 = undefined;
5840 // var len: usize = runtime_zero();
5841 // @memset(ptr[0..len], 42);
5842 //
5843 // So explicitly avoid using `memory.fill` in the `len == 0` case. Lovely design.
5844 try cg.emitWValue(len);
5845 try cg.addTag(.i32_eqz);
5846 try cg.addLabel(.br_if, 0);
5847 }
60215848
6022 const int_info = ty.intInfo(zcu);5849 try cg.lowerToStack(ptr);
6023 const wasm_bits = toWasmBits(int_info.bits) orelse {5850 try cg.emitWValue(value);
6024 return cg.fail("TODO: Implement `@mulWithOverflow` for integer bitsize: {d}", .{int_info.bits});5851 try cg.emitWValue(len);
6025 };5852 try cg.addExtended(.memory_fill);
60265853
6027 const zero: WValue = switch (wasm_bits) {5854 if (!len0_ok) {
6028 32 => .{ .imm32 = 0 },5855 try cg.endBlock();
6029 64, 128 => .{ .imm64 = 0 },5856 }
6030 else => unreachable,
6031 };
60325857
6033 // for 32 bit integers we upcast it to a 64bit integer5858 return;
6034 const mul = if (wasm_bits == 32) blk: {5859 }
6035 const new_ty = if (int_info.signedness == .signed) Type.i64 else Type.u64;
6036 const lhs_upcast = try cg.intcast(lhs, ty, new_ty);
6037 const rhs_upcast = try cg.intcast(rhs, ty, new_ty);
6038 const bin_op = try (try cg.binOp(lhs_upcast, rhs_upcast, new_ty, .mul)).toLocal(cg, new_ty);
6039 const res = try (try cg.trunc(bin_op, ty, new_ty)).toLocal(cg, ty);
6040 const res_upcast = try cg.intcast(res, ty, new_ty);
6041 _ = try cg.cmp(res_upcast, bin_op, new_ty, .neq);
6042 try cg.addLocal(.local_set, overflow_bit.local.value);
6043 break :blk res;
6044 } else if (wasm_bits == 64) blk: {
6045 const new_ty = if (int_info.signedness == .signed) Type.i128 else Type.u128;
6046 const lhs_upcast = try cg.intcast(lhs, ty, new_ty);
6047 const rhs_upcast = try cg.intcast(rhs, ty, new_ty);
6048 const bin_op = try (try cg.binOp(lhs_upcast, rhs_upcast, new_ty, .mul)).toLocal(cg, new_ty);
6049 const res = try (try cg.trunc(bin_op, ty, new_ty)).toLocal(cg, ty);
6050 const res_upcast = try cg.intcast(res, ty, new_ty);
6051 _ = try cg.cmp(res_upcast, bin_op, new_ty, .neq);
6052 try cg.addLocal(.local_set, overflow_bit.local.value);
6053 break :blk res;
6054 } else if (int_info.bits == 128 and int_info.signedness == .unsigned) blk: {
6055 var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);
6056 defer lhs_lsb.free(cg);
6057 var lhs_msb = try (try cg.load(lhs, Type.u64, 8)).toLocal(cg, Type.u64);
6058 defer lhs_msb.free(cg);
6059 var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);
6060 defer rhs_lsb.free(cg);
6061 var rhs_msb = try (try cg.load(rhs, Type.u64, 8)).toLocal(cg, Type.u64);
6062 defer rhs_msb.free(cg);
60635860
6064 const cross_1 = try cg.callIntrinsic(5861 const final_len: WValue = switch (len) {
6065 .__multi3,5862 .imm32 => |val| .{ .imm32 = val * abi_size },
6066 &[_]InternPool.Index{.i64_type} ** 4,5863 .imm64 => |val| .{ .imm64 = val * abi_size },
6067 Type.i128,5864 else => if (abi_size != 1) blk: {
6068 &.{ lhs_msb, zero, rhs_lsb, zero },5865 const new_len = try cg.ensureAllocLocal(Type.usize);
6069 );5866 try cg.emitWValue(len);
6070 const cross_2 = try cg.callIntrinsic(5867 switch (cg.ptr_size) {
6071 .__multi3,5868 .wasm32 => {
6072 &[_]InternPool.Index{.i64_type} ** 4,5869 try cg.emitWValue(.{ .imm32 = abi_size });
6073 Type.i128,5870 try cg.addTag(.i32_mul);
6074 &.{ rhs_msb, zero, lhs_lsb, zero },5871 },
6075 );5872 .wasm64 => {
6076 const mul_lsb = try cg.callIntrinsic(5873 try cg.emitWValue(.{ .imm64 = abi_size });
6077 .__multi3,5874 try cg.addTag(.i64_mul);
6078 &[_]InternPool.Index{.i64_type} ** 4,5875 },
6079 Type.i128,5876 }
6080 &.{ rhs_lsb, zero, lhs_lsb, zero },5877 try cg.addLocal(.local_set, new_len.local.value);
6081 );5878 break :blk new_len;
5879 } else len,
5880 };
60825881
6083 const rhs_msb_not_zero = try cg.cmp(rhs_msb, zero, Type.u64, .neq);5882 var end_ptr = try cg.allocLocal(Type.usize);
6084 const lhs_msb_not_zero = try cg.cmp(lhs_msb, zero, Type.u64, .neq);5883 defer end_ptr.free(cg);
6085 const both_msb_not_zero = try cg.binOp(rhs_msb_not_zero, lhs_msb_not_zero, Type.bool, .@"and");5884 var new_ptr = try cg.buildPointerOffset(ptr, 0, .new);
6086 const cross_1_msb = try cg.load(cross_1, Type.u64, 8);5885 defer new_ptr.free(cg);
6087 const cross_1_msb_not_zero = try cg.cmp(cross_1_msb, zero, Type.u64, .neq);
6088 const cond_1 = try cg.binOp(both_msb_not_zero, cross_1_msb_not_zero, Type.bool, .@"or");
6089 const cross_2_msb = try cg.load(cross_2, Type.u64, 8);
6090 const cross_2_msb_not_zero = try cg.cmp(cross_2_msb, zero, Type.u64, .neq);
6091 const cond_2 = try cg.binOp(cond_1, cross_2_msb_not_zero, Type.bool, .@"or");
60925886
6093 const cross_1_lsb = try cg.load(cross_1, Type.u64, 0);5887 // get the loop conditional: if current pointer address equals final pointer's address
6094 const cross_2_lsb = try cg.load(cross_2, Type.u64, 0);5888 try cg.lowerToStack(ptr);
6095 const cross_add = try cg.binOp(cross_1_lsb, cross_2_lsb, Type.u64, .add);5889 try cg.emitWValue(final_len);
5890 switch (cg.ptr_size) {
5891 .wasm32 => try cg.addTag(.i32_add),
5892 .wasm64 => try cg.addTag(.i64_add),
5893 }
5894 try cg.addLocal(.local_set, end_ptr.local.value);
60965895
6097 var mul_lsb_msb = try (try cg.load(mul_lsb, Type.u64, 8)).toLocal(cg, Type.u64);5896 // outer block to jump to when loop is done
6098 defer mul_lsb_msb.free(cg);5897 try cg.startBlock(.block, .empty);
6099 var all_add = try (try cg.binOp(cross_add, mul_lsb_msb, Type.u64, .add)).toLocal(cg, Type.u64);5898 try cg.startBlock(.loop, .empty);
6100 defer all_add.free(cg);
6101 const add_overflow = try cg.cmp(all_add, mul_lsb_msb, Type.u64, .lt);
61025899
6103 // result for overflow bit5900 // check for condition for loop end
6104 _ = try cg.binOp(cond_2, add_overflow, Type.bool, .@"or");5901 try cg.emitWValue(new_ptr);
6105 try cg.addLocal(.local_set, overflow_bit.local.value);5902 try cg.emitWValue(end_ptr);
5903 switch (cg.ptr_size) {
5904 .wasm32 => try cg.addTag(.i32_eq),
5905 .wasm64 => try cg.addTag(.i64_eq),
5906 }
5907 try cg.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
61065908
6107 const tmp_result = try cg.allocStack(Type.u128);5909 // store the value at the current position of the pointer
6108 try cg.emitWValue(tmp_result);5910 try cg.store(new_ptr, value, elem_ty, 0);
6109 const mul_lsb_lsb = try cg.load(mul_lsb, Type.u64, 0);
6110 try cg.store(.stack, mul_lsb_lsb, Type.u64, tmp_result.offset());
6111 try cg.store(tmp_result, all_add, Type.u64, 8);
6112 break :blk tmp_result;
6113 } else if (int_info.bits == 128 and int_info.signedness == .signed) blk: {
6114 const overflow_ret = try cg.allocStack(Type.i32);
6115 const res = try cg.callIntrinsic(
6116 .__muloti4,
6117 &[_]InternPool.Index{ .i128_type, .i128_type, .usize_type },
6118 Type.i128,
6119 &.{ lhs, rhs, overflow_ret },
6120 );
6121 _ = try cg.load(overflow_ret, Type.i32, 0);
6122 try cg.addLocal(.local_set, overflow_bit.local.value);
6123 break :blk res;
6124 } else return cg.fail("TODO: @mulWithOverflow for {f}", .{ty.fmt(pt)});
6125 var bin_op_local = try mul.toLocal(cg, ty);
6126 defer bin_op_local.free(cg);
61275911
6128 const result = try cg.allocStack(cg.typeOfIndex(inst));5912 // move the pointer to the next element
6129 const offset: u32 = @intCast(ty.abiSize(zcu));5913 try cg.emitWValue(new_ptr);
6130 try cg.store(result, bin_op_local, ty, 0);5914 switch (cg.ptr_size) {
6131 try cg.store(result, overflow_bit, Type.u1, offset);5915 .wasm32 => {
5916 try cg.emitWValue(.{ .imm32 = abi_size });
5917 try cg.addTag(.i32_add);
5918 },
5919 .wasm64 => {
5920 try cg.emitWValue(.{ .imm64 = abi_size });
5921 try cg.addTag(.i64_add);
5922 },
5923 }
5924 try cg.addLocal(.local_set, new_ptr.local.value);
61325925
6133 return cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs });5926 // end of loop
5927 try cg.addLabel(.br, 0); // jump to start of loop
5928 try cg.endBlock();
5929 try cg.endBlock();
6134}5930}
61355931
6136fn airMaxMin(5932fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6137 cg: *CodeGen,
6138 inst: Air.Inst.Index,
6139 op: enum { fmax, fmin },
6140 cmp_op: std.math.CompareOperator,
6141) InnerError!void {
6142 const zcu = cg.pt.zcu;5933 const zcu = cg.pt.zcu;
6143 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;5934 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
61445935
6145 const ty = cg.typeOfIndex(inst);5936 const array_ty = cg.typeOf(bin_op.lhs);
6146 if (ty.zigTypeTag(zcu) == .vector) {5937 const array = try cg.resolveInst(bin_op.lhs);
6147 return cg.fail("TODO: `@maximum` and `@minimum` for vectors", .{});5938 const index = try cg.resolveInst(bin_op.rhs);
6148 }5939 const elem_ty = array_ty.childType(zcu);
5940 const elem_size = elem_ty.abiSize(zcu);
61495941
6150 if (ty.abiSize(zcu) > 16) {5942 if (isByRef(array_ty, zcu, cg.target)) {
6151 return cg.fail("TODO: `@maximum` and `@minimum` for types larger than 16 bytes", .{});5943 try cg.lowerToStack(array);
6152 }5944 try cg.emitWValue(index);
5945 try cg.addImm32(@intCast(elem_size));
5946 try cg.addTag(.i32_mul);
5947 try cg.addTag(.i32_add);
5948 } else {
5949 assert(array_ty.zigTypeTag(zcu) == .vector);
61535950
6154 const lhs = try cg.resolveInst(bin_op.lhs);5951 switch (index) {
6155 const rhs = try cg.resolveInst(bin_op.rhs);5952 inline .imm32, .imm64 => |lane| {
5953 const opcode: std.wasm.SimdOpcode = switch (elem_ty.bitSize(zcu)) {
5954 8 => if (elem_ty.isSignedInt(zcu)) .i8x16_extract_lane_s else .i8x16_extract_lane_u,
5955 16 => if (elem_ty.isSignedInt(zcu)) .i16x8_extract_lane_s else .i16x8_extract_lane_u,
5956 32 => if (elem_ty.isInt(zcu)) .i32x4_extract_lane else .f32x4_extract_lane,
5957 64 => if (elem_ty.isInt(zcu)) .i64x2_extract_lane else .f64x2_extract_lane,
5958 else => unreachable,
5959 };
61565960
6157 if (ty.zigTypeTag(zcu) == .float) {5961 var operands = [_]u32{ @intFromEnum(opcode), @as(u8, @intCast(lane)) };
6158 const intrinsic = switch (op) {
6159 inline .fmin, .fmax => |ct_op| switch (ty.floatBits(cg.target)) {
6160 inline 16, 32, 64, 80, 128 => |bits| @field(
6161 Mir.Intrinsic,
6162 libcFloatPrefix(bits) ++ @tagName(ct_op) ++ libcFloatSuffix(bits),
6163 ),
6164 else => unreachable,
6165 },
6166 };
6167 const result = try cg.callIntrinsic(intrinsic, &.{ ty.ip_index, ty.ip_index }, ty, &.{ lhs, rhs });
6168 try cg.lowerToStack(result);
6169 } else {
6170 // operands to select from
6171 try cg.lowerToStack(lhs);
6172 try cg.lowerToStack(rhs);
6173 _ = try cg.cmp(lhs, rhs, ty, cmp_op);
61745962
6175 // based on the result from comparison, return operand 0 or 1.5963 try cg.emitWValue(array);
6176 try cg.addTag(.select);
6177 }
61785964
6179 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });5965 const extra_index: u32 = @intCast(cg.mir_extra.items.len);
6180}5966 try cg.mir_extra.appendSlice(cg.gpa, &operands);
5967 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
61815968
6182fn airMulAdd(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5969 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6183 const zcu = cg.pt.zcu;5970 },
6184 const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;5971 else => {
6185 const bin_op = cg.air.extraData(Air.Bin, pl_op.payload).data;5972 const stack_vec = try cg.allocStack(array_ty);
5973 try cg.store(stack_vec, array, array_ty, 0);
61865974
6187 const ty = cg.typeOfIndex(inst);5975 // Is a non-unrolled vector (v128)
6188 if (ty.zigTypeTag(zcu) == .vector) {5976 try cg.lowerToStack(stack_vec);
6189 return cg.fail("TODO: `@mulAdd` for vectors", .{});5977 try cg.emitWValue(index);
5978 try cg.addImm32(@intCast(elem_size));
5979 try cg.addTag(.i32_mul);
5980 try cg.addTag(.i32_add);
5981 },
5982 }
6190 }5983 }
61915984
6192 const addend = try cg.resolveInst(pl_op.operand);5985 const elem_result = if (isByRef(elem_ty, zcu, cg.target))
6193 const lhs = try cg.resolveInst(bin_op.lhs);5986 .stack
6194 const rhs = try cg.resolveInst(bin_op.rhs);5987 else
61955988 try cg.load(.stack, elem_ty, 0);
6196 const result = if (ty.floatBits(cg.target) == 16) fl_result: {
6197 const rhs_ext = try cg.fpext(rhs, ty, Type.f32);
6198 const lhs_ext = try cg.fpext(lhs, ty, Type.f32);
6199 const addend_ext = try cg.fpext(addend, ty, Type.f32);
6200 // call to compiler-rt `fn fmaf(f32, f32, f32) f32`
6201 const result = try cg.callIntrinsic(
6202 .fmaf,
6203 &.{ .f32_type, .f32_type, .f32_type },
6204 Type.f32,
6205 &.{ rhs_ext, lhs_ext, addend_ext },
6206 );
6207 break :fl_result try cg.fptrunc(result, Type.f32, ty);
6208 } else result: {
6209 const mul_result = try cg.binOp(lhs, rhs, ty, .mul);
6210 break :result try cg.binOp(mul_result, addend, ty, .add);
6211 };
62125989
6213 return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs, pl_op.operand });5990 return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
6214}5991}
62155992
6216fn airClz(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5993fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6217 const zcu = cg.pt.zcu;5994 const zcu = cg.pt.zcu;
6218 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5995 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6219
6220 const ty = cg.typeOf(ty_op.operand);
6221 if (ty.zigTypeTag(zcu) == .vector) {
6222 return cg.fail("TODO: `@clz` for vectors", .{});
6223 }
6224
6225 const operand = try cg.resolveInst(ty_op.operand);5996 const operand = try cg.resolveInst(ty_op.operand);
6226 const int_info = ty.intInfo(zcu);5997 const ty = cg.typeOfIndex(inst);
6227 const wasm_bits = toWasmBits(int_info.bits) orelse {5998 const elem_ty = ty.childType(zcu);
6228 return cg.fail("TODO: `@clz` for integers with bitsize '{d}'", .{int_info.bits});
6229 };
62305999
6231 switch (wasm_bits) {6000 if (determineSimdStoreStrategy(ty, zcu, cg.target) == .direct) blk: {
6232 32 => {6001 switch (operand) {
6233 if (int_info.signedness == .signed) {6002 // when the operand lives in the linear memory section, we can directly
6234 const mask = ~@as(u32, 0) >> @intCast(32 - int_info.bits);6003 // load and splat the value at once. Meaning we do not first have to load
6235 _ = try cg.binOp(operand, .{ .imm32 = mask }, ty, .@"and");6004 // the scalar value onto the stack.
6236 } else {6005 .stack_offset, .nav_ref, .uav_ref => {
6006 const opcode = switch (elem_ty.bitSize(zcu)) {
6007 8 => @intFromEnum(std.wasm.SimdOpcode.v128_load8_splat),
6008 16 => @intFromEnum(std.wasm.SimdOpcode.v128_load16_splat),
6009 32 => @intFromEnum(std.wasm.SimdOpcode.v128_load32_splat),
6010 64 => @intFromEnum(std.wasm.SimdOpcode.v128_load64_splat),
6011 else => break :blk, // Cannot make use of simd-instructions
6012 };
6237 try cg.emitWValue(operand);6013 try cg.emitWValue(operand);
6238 }6014 const extra_index: u32 = @intCast(cg.mir_extra.items.len);
6239 try cg.addTag(.i32_clz);6015 // stores as := opcode, offset, alignment (opcode::memarg)
6240 },6016 try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
6241 64 => {6017 opcode,
6242 if (int_info.signedness == .signed) {6018 operand.offset(),
6243 const mask = ~@as(u64, 0) >> @intCast(64 - int_info.bits);6019 @intCast(elem_ty.abiAlignment(zcu).toByteUnits().?),
6244 _ = try cg.binOp(operand, .{ .imm64 = mask }, ty, .@"and");6020 });
6245 } else {6021 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
6022 return cg.finishAir(inst, .stack, &.{ty_op.operand});
6023 },
6024 .local => {
6025 const opcode = switch (elem_ty.bitSize(zcu)) {
6026 8 => @intFromEnum(std.wasm.SimdOpcode.i8x16_splat),
6027 16 => @intFromEnum(std.wasm.SimdOpcode.i16x8_splat),
6028 32 => if (elem_ty.isInt(zcu)) @intFromEnum(std.wasm.SimdOpcode.i32x4_splat) else @intFromEnum(std.wasm.SimdOpcode.f32x4_splat),
6029 64 => if (elem_ty.isInt(zcu)) @intFromEnum(std.wasm.SimdOpcode.i64x2_splat) else @intFromEnum(std.wasm.SimdOpcode.f64x2_splat),
6030 else => break :blk, // Cannot make use of simd-instructions
6031 };
6246 try cg.emitWValue(operand);6032 try cg.emitWValue(operand);
6247 }6033 const extra_index: u32 = @intCast(cg.mir_extra.items.len);
6248 try cg.addTag(.i64_clz);6034 try cg.mir_extra.append(cg.gpa, opcode);
6249 try cg.addTag(.i32_wrap_i64);6035 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
6250 },6036 return cg.finishAir(inst, .stack, &.{ty_op.operand});
6251 128 => {6037 },
6252 var msb = try (try cg.load(operand, Type.u64, 8)).toLocal(cg, Type.u64);6038 else => unreachable,
6253 defer msb.free(cg);6039 }
62546040 }
6255 try cg.emitWValue(msb);6041 const elem_size = elem_ty.bitSize(zcu);
6256 try cg.addTag(.i64_clz);6042 const vector_len = @as(usize, @intCast(ty.vectorLen(zcu)));
6257 _ = try cg.load(operand, Type.u64, 0);6043 if ((!std.math.isPowerOfTwo(elem_size) or elem_size % 8 != 0) and vector_len > 1) {
6258 try cg.addTag(.i64_clz);6044 return cg.fail("TODO: WebAssembly `@splat` for arbitrary element bitsize {d}", .{elem_size});
6259 try cg.emitWValue(.{ .imm64 = 64 });
6260 try cg.addTag(.i64_add);
6261 _ = try cg.cmp(msb, .{ .imm64 = 0 }, Type.u64, .neq);
6262 try cg.addTag(.select);
6263 try cg.addTag(.i32_wrap_i64);
6264 },
6265 else => unreachable,
6266 }6045 }
62676046
6268 if (wasm_bits != int_info.bits) {6047 const result = try cg.allocStack(ty);
6269 try cg.emitWValue(.{ .imm32 = wasm_bits - int_info.bits });6048 const elem_byte_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
6270 try cg.addTag(.i32_sub);6049 var index: usize = 0;
6050 var offset: u32 = 0;
6051 while (index < vector_len) : (index += 1) {
6052 try cg.store(result, operand, elem_ty, offset);
6053 offset += elem_byte_size;
6271 }6054 }
62726055
6273 return cg.finishAir(inst, .stack, &.{ty_op.operand});6056 return cg.finishAir(inst, result, &.{ty_op.operand});
6057}
6058
6059fn airSelect(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6060 const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6061 const operand = try cg.resolveInst(pl_op.operand);
6062
6063 _ = operand;
6064 return cg.fail("TODO: Implement wasm airSelect", .{});
6274}6065}
62756066
6276fn airCtz(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {6067fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6277 const zcu = cg.pt.zcu;6068 const pt = cg.pt;
6278 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6069 const zcu = pt.zcu;
6070
6071 const unwrapped = cg.air.unwrapShuffleOne(zcu, inst);
6072 const result_ty = unwrapped.result_ty;
6073 const mask = unwrapped.mask;
6074 const operand = try cg.resolveInst(unwrapped.operand);
6075
6076 const elem_ty = result_ty.childType(zcu);
6077 const elem_size = elem_ty.abiSize(zcu);
6078
6079 // TODO: this function could have an `i8x16_shuffle` fast path like `airShuffleTwo` if we were
6080 // to lower the comptime-known operands to a non-by-ref vector value.
62796081
6280 const ty = cg.typeOf(ty_op.operand);6082 // TODO: this is incorrect if either operand or the result is *not* by-ref, which is possible.
6083 // I tried to fix it, but I couldn't make much sense of how this backend handles memory.
6084 if (!isByRef(result_ty, zcu, cg.target) or
6085 !isByRef(cg.typeOf(unwrapped.operand), zcu, cg.target)) return cg.fail("TODO: handle mixed by-ref shuffle", .{});
62816086
6282 if (ty.zigTypeTag(zcu) == .vector) {6087 const dest_alloc = try cg.allocStack(result_ty);
6283 return cg.fail("TODO: `@ctz` for vectors", .{});6088 for (mask, 0..) |mask_elem, out_idx| {
6089 try cg.emitWValue(dest_alloc);
6090 const elem_val = switch (mask_elem.unwrap()) {
6091 .elem => |idx| try cg.load(operand, elem_ty, @intCast(elem_size * idx)),
6092 .value => |val| try cg.lowerConstant(.fromInterned(val)),
6093 };
6094 try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));
6284 }6095 }
6096 return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand});
6097}
62856098
6286 const operand = try cg.resolveInst(ty_op.operand);6099fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6287 const int_info = ty.intInfo(zcu);6100 const pt = cg.pt;
6288 const wasm_bits = toWasmBits(int_info.bits) orelse {6101 const zcu = pt.zcu;
6289 return cg.fail("TODO: `@clz` for integers with bitsize '{d}'", .{int_info.bits});
6290 };
62916102
6292 switch (wasm_bits) {6103 const unwrapped = cg.air.unwrapShuffleTwo(zcu, inst);
6293 32 => {6104 const result_ty = unwrapped.result_ty;
6294 if (wasm_bits != int_info.bits) {6105 const mask = unwrapped.mask;
6295 const val: u32 = @as(u32, 1) << @as(u5, @intCast(int_info.bits));6106 const operand_a = try cg.resolveInst(unwrapped.operand_a);
6296 // leave value on the stack6107 const operand_b = try cg.resolveInst(unwrapped.operand_b);
6297 _ = try cg.binOp(operand, .{ .imm32 = val }, ty, .@"or");
6298 } else try cg.emitWValue(operand);
6299 try cg.addTag(.i32_ctz);
6300 },
6301 64 => {
6302 if (wasm_bits != int_info.bits) {
6303 const val: u64 = @as(u64, 1) << @as(u6, @intCast(int_info.bits));
6304 // leave value on the stack
6305 _ = try cg.binOp(operand, .{ .imm64 = val }, ty, .@"or");
6306 } else try cg.emitWValue(operand);
6307 try cg.addTag(.i64_ctz);
6308 try cg.addTag(.i32_wrap_i64);
6309 },
6310 128 => {
6311 var lsb = try (try cg.load(operand, Type.u64, 0)).toLocal(cg, Type.u64);
6312 defer lsb.free(cg);
63136108
6314 try cg.emitWValue(lsb);6109 const a_ty = cg.typeOf(unwrapped.operand_a);
6315 try cg.addTag(.i64_ctz);6110 const b_ty = cg.typeOf(unwrapped.operand_b);
6316 _ = try cg.load(operand, Type.u64, 8);6111 const elem_ty = result_ty.childType(zcu);
6317 if (wasm_bits != int_info.bits) {6112 const elem_size = elem_ty.abiSize(zcu);
6318 try cg.addImm64(@as(u64, 1) << @as(u6, @intCast(int_info.bits - 64)));6113
6319 try cg.addTag(.i64_or);6114 // WASM has `i8x16_shuffle`, which we can apply if the element type bit size is a multiple of 8
6320 }6115 // and the input and output vectors have a bit size of 128 (and are hence not by-ref). Otherwise,
6321 try cg.addTag(.i64_ctz);6116 // we fall back to a naive loop lowering.
6322 try cg.addImm64(64);6117 if (!isByRef(a_ty, zcu, cg.target) and
6323 if (wasm_bits != int_info.bits) {6118 !isByRef(b_ty, zcu, cg.target) and
6324 try cg.addTag(.i64_or);6119 !isByRef(result_ty, zcu, cg.target) and
6325 } else {6120 elem_ty.bitSize(zcu) % 8 == 0)
6326 try cg.addTag(.i64_add);6121 {
6122 var lane_map: [16]u8 align(4) = undefined;
6123 const lanes_per_elem: usize = @intCast(elem_ty.bitSize(zcu) / 8);
6124 for (mask, 0..) |mask_elem, out_idx| {
6125 const out_first_lane = out_idx * lanes_per_elem;
6126 const in_first_lane = switch (mask_elem.unwrap()) {
6127 .a_elem => |i| i * lanes_per_elem,
6128 .b_elem => |i| i * lanes_per_elem + 16,
6129 .undef => 0, // doesn't matter
6130 };
6131 for (lane_map[out_first_lane..][0..lanes_per_elem], in_first_lane..) |*out, in| {
6132 out.* = @intCast(in);
6327 }6133 }
6328 _ = try cg.cmp(lsb, .{ .imm64 = 0 }, Type.u64, .neq);6134 }
6329 try cg.addTag(.select);6135 try cg.emitWValue(operand_a);
6330 try cg.addTag(.i32_wrap_i64);6136 try cg.emitWValue(operand_b);
6331 },6137 const extra_index: u32 = @intCast(cg.mir_extra.items.len);
6332 else => unreachable,6138 try cg.mir_extra.appendSlice(cg.gpa, &.{
6139 @intFromEnum(std.wasm.SimdOpcode.i8x16_shuffle),
6140 @bitCast(lane_map[0..4].*),
6141 @bitCast(lane_map[4..8].*),
6142 @bitCast(lane_map[8..12].*),
6143 @bitCast(lane_map[12..].*),
6144 });
6145 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
6146 return cg.finishAir(inst, .stack, &.{ unwrapped.operand_a, unwrapped.operand_b });
6333 }6147 }
63346148
6335 return cg.finishAir(inst, .stack, &.{ty_op.operand});6149 // TODO: this is incorrect if either operand or the result is *not* by-ref, which is possible.
6336}6150 // I tried to fix it, but I couldn't make much sense of how this backend handles memory.
6151 if (!isByRef(result_ty, zcu, cg.target) or
6152 !isByRef(a_ty, zcu, cg.target) or
6153 !isByRef(b_ty, zcu, cg.target)) return cg.fail("TODO: handle mixed by-ref shuffle", .{});
63376154
6338fn airDbgStmt(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {6155 const dest_alloc = try cg.allocStack(result_ty);
6339 const dbg_stmt = cg.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;6156 for (mask, 0..) |mask_elem, out_idx| {
6340 try cg.addInst(.{ .tag = .dbg_line, .data = .{6157 try cg.emitWValue(dest_alloc);
6341 .payload = try cg.addExtra(Mir.DbgLineColumn{6158 const elem_val = switch (mask_elem.unwrap()) {
6342 .line = dbg_stmt.line,6159 .a_elem => |idx| try cg.load(operand_a, elem_ty, @intCast(elem_size * idx)),
6343 .column = dbg_stmt.column,6160 .b_elem => |idx| try cg.load(operand_b, elem_ty, @intCast(elem_size * idx)),
6344 }),6161 .undef => try cg.emitUndefined(elem_ty),
6345 } });6162 };
6346 return cg.finishAir(inst, .none, &.{});6163 try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));
6164 }
6165 return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b });
6347}6166}
63486167
6349fn airDbgInlineBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {6168fn airReduce(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6350 const block = cg.air.unwrapDbgBlock(inst);6169 const reduce = cg.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
6351 // TODO6170 const operand = try cg.resolveInst(reduce.operand);
6352 try cg.lowerBlock(inst, block.ty, block.body);
6353}
63546171
6355fn airDbgVar(6172 _ = operand;
6356 cg: *CodeGen,6173 return cg.fail("TODO: Implement wasm airReduce", .{});
6357 inst: Air.Inst.Index,
6358 local_tag: link.File.Dwarf.WipNav.LocalVarTag,
6359 is_ptr: bool,
6360) InnerError!void {
6361 _ = is_ptr;
6362 _ = local_tag;
6363 return cg.finishAir(inst, .none, &.{});
6364}6174}
63656175
6366fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {6176fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6367 const unwrapped_try = cg.air.unwrapTry(inst);6177 const pt = cg.pt;
6368 const body = unwrapped_try.else_body;6178 const zcu = pt.zcu;
6369 const err_union = try cg.resolveInst(unwrapped_try.error_union);6179 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6370 const err_union_ty = cg.typeOf(unwrapped_try.error_union);6180 const result_ty = cg.typeOfIndex(inst);
6371 const result = try lowerTry(cg, inst, err_union, body, err_union_ty, false);6181 const len = @as(usize, @intCast(result_ty.arrayLen(zcu)));
6372 return cg.finishAir(inst, result, &.{unwrapped_try.error_union});6182 const elements: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..len]);
6373}
63746183
6375fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {6184 const result: WValue = result_value: {
6376 const zcu = cg.pt.zcu;6185 switch (result_ty.zigTypeTag(zcu)) {
6377 const unwrapped_try = cg.air.unwrapTryPtr(inst);6186 .array => {
6378 const err_union_ptr = try cg.resolveInst(unwrapped_try.error_union_ptr);6187 const result = try cg.allocStack(result_ty);
6379 const body = unwrapped_try.else_body;6188 const elem_ty = result_ty.childType(zcu);
6380 const err_union_ty = cg.typeOf(unwrapped_try.error_union_ptr).childType(zcu);6189 const elem_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
6381 const result = try lowerTry(cg, inst, err_union_ptr, body, err_union_ty, true);6190 const sentinel = result_ty.sentinel(zcu);
6382 return cg.finishAir(inst, result, &.{unwrapped_try.error_union_ptr});
6383}
63846191
6385fn lowerTry(6192 // When the element type is by reference, we must copy the entire
6386 cg: *CodeGen,6193 // value. It is therefore safer to move the offset pointer and store
6387 inst: Air.Inst.Index,6194 // each value individually, instead of using store offsets.
6388 err_union: WValue,6195 if (isByRef(elem_ty, zcu, cg.target)) {
6389 body: []const Air.Inst.Index,6196 // copy stack pointer into a temporary local, which is
6390 err_union_ty: Type,6197 // moved for each element to store each value in the right position.
6391 operand_is_ptr: bool,6198 const offset = try cg.buildPointerOffset(result, 0, .new);
6392) InnerError!WValue {6199 for (elements, 0..) |elem, elem_index| {
6393 const zcu = cg.pt.zcu;6200 const elem_val = try cg.resolveInst(elem);
6201 try cg.store(offset, elem_val, elem_ty, 0);
63946202
6395 const pl_ty = err_union_ty.errorUnionPayload(zcu);6203 if (elem_index < elements.len - 1 or sentinel != null) {
6396 const pl_has_bits = pl_ty.hasRuntimeBits(zcu);6204 _ = try cg.buildPointerOffset(offset, elem_size, .modify);
6205 }
6206 }
6207 if (sentinel) |s| {
6208 const val = try cg.resolveValue(s);
6209 try cg.store(offset, val, elem_ty, 0);
6210 }
6211 } else {
6212 var offset: u32 = 0;
6213 for (elements) |elem| {
6214 const elem_val = try cg.resolveInst(elem);
6215 try cg.store(result, elem_val, elem_ty, offset);
6216 offset += elem_size;
6217 }
6218 if (sentinel) |s| {
6219 const val = try cg.resolveValue(s);
6220 try cg.store(result, val, elem_ty, offset);
6221 }
6222 }
6223 break :result_value result;
6224 },
6225 .@"struct" => switch (result_ty.containerLayout(zcu)) {
6226 .@"packed" => unreachable, // legalize .expand_packed_aggregate_init
6227 else => {
6228 const result = try cg.allocStack(result_ty);
6229 const offset = try cg.buildPointerOffset(result, 0, .new); // pointer to offset
6230 var prev_field_offset: u64 = 0;
6231 for (elements, 0..) |elem, elem_index| {
6232 if (try result_ty.structFieldValueComptime(pt, elem_index) != null) continue;
63976233
6398 if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {6234 const elem_ty = result_ty.fieldType(elem_index, zcu);
6399 // Block we can jump out of when error is not set6235 const field_offset = result_ty.structFieldOffset(elem_index, zcu);
6400 try cg.startBlock(.block, .empty);6236 _ = try cg.buildPointerOffset(offset, @intCast(field_offset - prev_field_offset), .modify);
6237 prev_field_offset = field_offset;
64016238
6402 // check if the error tag is set for the error union.6239 const value = try cg.resolveInst(elem);
6403 try cg.emitWValue(err_union);6240 try cg.store(offset, value, elem_ty, 0);
6404 if (pl_has_bits or operand_is_ptr) {6241 }
6405 const err_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
6406 try cg.addMemArg(.i32_load16_u, .{
6407 .offset = err_union.offset() + err_offset,
6408 .alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),
6409 });
6410 }
6411 try cg.addTag(.i32_eqz);
6412 try cg.addLabel(.br_if, 0); // jump out of block when error is '0'
64136242
6414 const liveness = cg.liveness.getCondBr(inst);6243 break :result_value result;
6415 try cg.branches.append(cg.gpa, .{});6244 },
6416 try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.else_deaths.len + liveness.then_deaths.len);6245 },
6417 defer {6246 .vector => return cg.fail("TODO: Wasm backend: implement airAggregateInit for vectors", .{}),
6418 var branch = cg.branches.pop().?;6247 else => unreachable,
6419 branch.deinit(cg.gpa);
6420 }6248 }
6421 try cg.genBody(body);6249 };
6422 try cg.endBlock();
6423 }
6424
6425 // if we reach here it means error was not set, and we want the payload
6426 if (!pl_has_bits and !operand_is_ptr) {
6427 return .none;
6428 }
64296250
6430 const pl_offset: u32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));6251 if (elements.len <= Air.Liveness.bpi - 1) {
6431 if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) {6252 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
6432 return buildPointerOffset(cg, err_union, pl_offset, .new);6253 @memcpy(buf[0..elements.len], elements);
6254 return cg.finishAir(inst, result, &buf);
6433 }6255 }
6434 const payload = try cg.load(err_union, pl_ty, pl_offset);6256 var bt = try cg.iterateBigTomb(inst, elements.len);
6435 return payload.toLocal(cg, pl_ty);6257 for (elements) |arg| bt.feed(arg);
6258 return bt.finishAir(result);
6436}6259}
64376260
6438fn airByteSwap(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {6261fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6439 const zcu = cg.pt.zcu;6262 const pt = cg.pt;
6440 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6263 const zcu = pt.zcu;
6264 const ip = &zcu.intern_pool;
6265 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6266 const extra = cg.air.extraData(Air.UnionInit, ty_pl.payload).data;
64416267
6442 const ty = cg.typeOfIndex(inst);6268 const result = result: {
6443 const operand = try cg.resolveInst(ty_op.operand);6269 const union_ty = cg.typeOfIndex(inst);
6270 const layout = union_ty.unionGetLayout(zcu);
6271 const union_obj = zcu.typeToUnion(union_ty).?;
6272 const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
6273 const field_name = ip.loadEnumType(union_obj.enum_tag_type).field_names.get(ip)[extra.field_index];
64446274
6445 if (ty.zigTypeTag(zcu) == .vector) {6275 const tag_int = blk: {
6446 return cg.fail("TODO: @byteSwap for vectors", .{});6276 const tag_ty = union_ty.unionTagTypeHypothetical(zcu);
6447 }6277 const enum_field_index = tag_ty.enumFieldIndex(field_name, zcu).?;
6448 const int_info = ty.intInfo(zcu);6278 const tag_val = try pt.enumValueFieldIndex(tag_ty, enum_field_index);
6449 const wasm_bits = toWasmBits(int_info.bits) orelse {6279 break :blk try cg.lowerConstant(tag_val);
6450 return cg.fail("TODO: @byteSwap for integers with bitsize {d}", .{int_info.bits});6280 };
6451 };6281 if (layout.payload_size == 0) {
6282 if (layout.tag_size == 0) {
6283 break :result .none;
6284 }
6285 assert(!isByRef(union_ty, zcu, cg.target));
6286 break :result tag_int;
6287 }
64526288
6453 // bytes are no-op6289 if (isByRef(union_ty, zcu, cg.target)) {
6454 if (int_info.bits == 8) {6290 const result_ptr = try cg.allocStack(union_ty);
6455 return cg.finishAir(inst, cg.reuseOperand(ty_op.operand, operand), &.{ty_op.operand});6291 const payload = try cg.resolveInst(extra.init);
6456 }6292 if (layout.tag_align.compare(.gte, layout.payload_align)) {
6293 if (isByRef(field_ty, zcu, cg.target)) {
6294 const payload_ptr = try cg.buildPointerOffset(result_ptr, layout.tag_size, .new);
6295 try cg.store(payload_ptr, payload, field_ty, 0);
6296 } else {
6297 try cg.store(result_ptr, payload, field_ty, @intCast(layout.tag_size));
6298 }
64576299
6458 const result = result: {6300 if (layout.tag_size > 0) {
6459 switch (wasm_bits) {6301 try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), 0);
6460 32 => {6302 }
6461 const intrin_ret = try cg.callIntrinsic(6303 } else {
6462 .__bswapsi2,6304 try cg.store(result_ptr, payload, field_ty, 0);
6463 &.{.u32_type},6305 if (layout.tag_size > 0) {
6464 Type.u32,6306 try cg.store(
6465 &.{operand},6307 result_ptr,
6466 );6308 tag_int,
6467 break :result if (int_info.bits == 32)6309 .fromInterned(union_obj.enum_tag_type),
6468 intrin_ret6310 @intCast(layout.payload_size),
6469 else6311 );
6470 try cg.binOp(intrin_ret, .{ .imm32 = 32 - int_info.bits }, ty, .shr);6312 }
6471 },6313 }
6472 64 => {6314 break :result result_ptr;
6473 const intrin_ret = try cg.callIntrinsic(6315 } else {
6474 .__bswapdi2,6316 const operand = try cg.resolveInst(extra.init);
6475 &.{.u64_type},6317 break :result (try cg.bitcast(union_ty, field_ty, operand)) orelse cg.reuseOperand(extra.init, operand);
6476 Type.u64,
6477 &.{operand},
6478 );
6479 break :result if (int_info.bits == 64)
6480 intrin_ret
6481 else
6482 try cg.binOp(intrin_ret, .{ .imm64 = 64 - int_info.bits }, ty, .shr);
6483 },
6484 else => return cg.fail("TODO: @byteSwap for integers with bitsize {d}", .{int_info.bits}),
6485 }6318 }
6486 };6319 };
6487 return cg.finishAir(inst, result, &.{ty_op.operand});6320
6321 return cg.finishAir(inst, result, &.{extra.init});
6488}6322}
64896323
6490fn airDiv(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {6324fn airPrefetch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6491 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;6325 const prefetch = cg.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;
6326 return cg.finishAir(inst, .none, &.{prefetch.ptr});
6327}
64926328
6493 const ty = cg.typeOfIndex(inst);6329fn airWasmMemorySize(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6494 const lhs = try cg.resolveInst(bin_op.lhs);6330 const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6495 const rhs = try cg.resolveInst(bin_op.rhs);
64966331
6497 const result = try cg.binOp(lhs, rhs, ty, .div);6332 try cg.addLabel(.memory_size, pl_op.payload);
6498 return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });6333 return cg.finishAir(inst, .stack, &.{pl_op.operand});
6499}6334}
65006335
6501fn airDivTrunc(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {6336fn airWasmMemoryGrow(cg: *CodeGen, inst: Air.Inst.Index) !void {
6502 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;6337 const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
65036338
6504 const ty = cg.typeOfIndex(inst);6339 const operand = try cg.resolveInst(pl_op.operand);
6505 const lhs = try cg.resolveInst(bin_op.lhs);6340 try cg.emitWValue(operand);
6506 const rhs = try cg.resolveInst(bin_op.rhs);6341 try cg.addLabel(.memory_grow, pl_op.payload);
6342 return cg.finishAir(inst, .stack, &.{pl_op.operand});
6343}
65076344
6508 const div_result = try cg.binOp(lhs, rhs, ty, .div);6345fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6346 const pt = cg.pt;
6347 const zcu = pt.zcu;
6348 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6349 const un_ty = cg.typeOf(bin_op.lhs).childType(zcu);
6350 const tag_ty = cg.typeOf(bin_op.rhs);
6351 const layout = un_ty.unionGetLayout(zcu);
6352 if (layout.tag_size == 0) return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
65096353
6510 if (ty.isAnyFloat()) {6354 const union_ptr = try cg.resolveInst(bin_op.lhs);
6511 const trunc_result = try cg.floatOp(.trunc, ty, &.{div_result});6355 const new_tag = try cg.resolveInst(bin_op.rhs);
6512 return cg.finishAir(inst, trunc_result, &.{ bin_op.lhs, bin_op.rhs });6356 if (layout.payload_size == 0) {
6357 try cg.store(union_ptr, new_tag, tag_ty, 0);
6358 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
6513 }6359 }
65146360
6515 return cg.finishAir(inst, div_result, &.{ bin_op.lhs, bin_op.rhs });6361 // when the tag alignment is smaller than the payload, the field will be stored
6362 // after the payload.
6363 const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align)) blk: {
6364 break :blk @intCast(layout.payload_size);
6365 } else 0;
6366 try cg.store(union_ptr, new_tag, tag_ty, offset);
6367 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
6516}6368}
65176369
6518fn airDivFloor(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {6370fn airGetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6519 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6520
6521 const zcu = cg.pt.zcu;6371 const zcu = cg.pt.zcu;
6522 const ty = cg.typeOfIndex(inst);6372 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6523 const lhs = try cg.resolveInst(bin_op.lhs);
6524 const rhs = try cg.resolveInst(bin_op.rhs);
6525
6526 if (ty.isUnsignedInt(zcu)) {
6527 _ = try cg.binOp(lhs, rhs, ty, .div);
6528 } else if (ty.isSignedInt(zcu)) {
6529 const int_bits = ty.intInfo(zcu).bits;
6530 const wasm_bits = toWasmBits(int_bits) orelse {
6531 return cg.fail("TODO: `@divFloor` for signed integers larger than 64 bits ({d} bits requested)", .{int_bits});
6532 };
6533
6534 if (wasm_bits > 64) {
6535 return cg.fail("TODO: `@divFloor` for signed integers larger than 64 bits ({d} bits requested)", .{int_bits});
6536 }
6537
6538 const zero: WValue = switch (wasm_bits) {
6539 32 => .{ .imm32 = 0 },
6540 64 => .{ .imm64 = 0 },
6541 else => unreachable,
6542 };
6543
6544 // tee leaves the value on the stack and stores it in a local.
6545 const quotient = try cg.allocLocal(ty);
6546 _ = try cg.binOp(lhs, rhs, ty, .div);
6547 try cg.addLocal(.local_tee, quotient.local.value);
6548
6549 // select takes a 32 bit value as the condition, so in the 64 bit case we use eqz to narrow
6550 // the 64 bit value we want to use as the condition to 32 bits.
6551 // This also inverts the condition (non 0 => 0, 0 => 1), so we put the adjusted and
6552 // non-adjusted quotients on the stack in the opposite order for 32 vs 64 bits.
6553 if (wasm_bits == 64) {
6554 try cg.emitWValue(quotient);
6555 }
65566373
6557 // 0 if the signs of rhs_wasm and lhs_wasm are the same, 1 otherwise.6374 const un_ty = cg.typeOf(ty_op.operand);
6558 _ = try cg.binOp(lhs, rhs, ty, .xor);6375 const tag_ty = cg.typeOfIndex(inst);
6559 _ = try cg.cmp(.stack, zero, ty, .lt);6376 const layout = un_ty.unionGetLayout(zcu);
6377 if (layout.tag_size == 0) return cg.finishAir(inst, .none, &.{ty_op.operand});
65606378
6561 switch (wasm_bits) {6379 const operand = try cg.resolveInst(ty_op.operand);
6562 32 => {6380 // when the tag alignment is smaller than the payload, the field will be stored
6563 try cg.addTag(.i32_sub);6381 // after the payload.
6564 try cg.emitWValue(quotient);6382 const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align))
6565 },6383 @intCast(layout.payload_size)
6566 64 => {6384 else
6567 try cg.addTag(.i64_extend_i32_u);6385 0;
6568 try cg.addTag(.i64_sub);6386 const result = try cg.load(operand, tag_ty, offset);
6569 },6387 return cg.finishAir(inst, result, &.{ty_op.operand});
6570 else => unreachable,6388}
6571 }
65726389
6573 _ = try cg.binOp(lhs, rhs, ty, .rem);6390fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6391 const zcu = cg.pt.zcu;
6392 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
65746393
6575 if (wasm_bits == 64) {6394 const err_set_ty = cg.typeOf(ty_op.operand).childType(zcu);
6576 try cg.addTag(.i64_eqz);6395 const payload_ty = err_set_ty.errorUnionPayload(zcu);
6577 }6396 const operand = try cg.resolveInst(ty_op.operand);
65786397
6579 try cg.addTag(.select);6398 // set error-tag to '0' to annotate error union is non-error
6399 try cg.store(
6400 operand,
6401 .{ .imm32 = 0 },
6402 Type.anyerror,
6403 @intCast(errUnionErrorOffset(payload_ty, zcu)),
6404 );
65806405
6581 // We need to zero the high bits because N bit comparisons consider all 32 or 64 bits, and6406 const result = result: {
6582 // expect all but the lowest N bits to be 0.6407 if (!payload_ty.hasRuntimeBits(zcu)) {
6583 // TODO: Should we be zeroing the high bits here or should we be ignoring the high bits6408 break :result cg.reuseOperand(ty_op.operand, operand);
6584 // when performing comparisons?
6585 if (int_bits != wasm_bits) {
6586 _ = try cg.wrapOperand(.stack, ty);
6587 }
6588 } else {
6589 const float_bits = ty.floatBits(cg.target);
6590 if (float_bits > 64) {
6591 return cg.fail("TODO: `@divFloor` for floats with bitsize: {d}", .{float_bits});
6592 }6409 }
6593 const is_f16 = float_bits == 16;
65946410
6595 const lhs_wasm = if (is_f16) try cg.fpext(lhs, Type.f16, Type.f32) else lhs;6411 break :result try cg.buildPointerOffset(operand, @as(u32, @intCast(errUnionPayloadOffset(payload_ty, zcu))), .new);
6596 const rhs_wasm = if (is_f16) try cg.fpext(rhs, Type.f16, Type.f32) else rhs;6412 };
6413 return cg.finishAir(inst, result, &.{ty_op.operand});
6414}
65976415
6598 try cg.emitWValue(lhs_wasm);6416fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6599 try cg.emitWValue(rhs_wasm);6417 const pt = cg.pt;
6418 const zcu = pt.zcu;
6419 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6420 const extra = cg.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
66006421
6601 switch (float_bits) {6422 const field_ptr = try cg.resolveInst(extra.field_ptr);
6602 16, 32 => {6423 const parent_ptr_ty = cg.typeOfIndex(inst);
6603 try cg.addTag(.f32_div);6424 const parent_ty = parent_ptr_ty.childType(zcu);
6604 try cg.addTag(.f32_floor);6425 const field_ptr_ty = cg.typeOf(extra.field_ptr);
6605 },6426 const field_index = extra.field_index;
6606 64 => {6427 const field_offset = switch (parent_ty.containerLayout(zcu)) {
6607 try cg.addTag(.f64_div);6428 .auto, .@"extern" => parent_ty.structFieldOffset(field_index, zcu),
6608 try cg.addTag(.f64_floor);6429 .@"packed" => offset: {
6609 },6430 const parent_ptr_offset = parent_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset;
6610 else => unreachable,6431 const field_offset = if (zcu.typeToStruct(parent_ty)) |loaded_struct| zcu.structPackedFieldBitOffset(loaded_struct, field_index) else 0;
6611 }6432 const field_ptr_offset = field_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset;
6433 break :offset @divExact(parent_ptr_offset + field_offset - field_ptr_offset, 8);
6434 },
6435 };
66126436
6613 if (is_f16) {6437 const result = if (field_offset != 0) result: {
6614 _ = try cg.fptrunc(.stack, Type.f32, Type.f16);6438 const base = try cg.buildPointerOffset(field_ptr, 0, .new);
6615 }6439 try cg.addLocal(.local_get, base.local.value);
6616 }6440 try cg.addImm32(@intCast(field_offset));
6441 try cg.addTag(.i32_sub);
6442 try cg.addLocal(.local_set, base.local.value);
6443 break :result base;
6444 } else cg.reuseOperand(extra.field_ptr, field_ptr);
66176445
6618 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });6446 return cg.finishAir(inst, result, &.{extra.field_ptr});
6619}6447}
66206448
6621fn airRem(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {6449fn sliceOrArrayPtr(cg: *CodeGen, ptr: WValue, ptr_ty: Type) InnerError!WValue {
6622 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;6450 const zcu = cg.pt.zcu;
66236451 if (ptr_ty.isSlice(zcu)) {
6624 const ty = cg.typeOfIndex(inst);6452 return cg.slicePtr(ptr);
6625 const lhs = try cg.resolveInst(bin_op.lhs);6453 } else {
6626 const rhs = try cg.resolveInst(bin_op.rhs);6454 return ptr;
66276455 }
6628 const result = try cg.binOp(lhs, rhs, ty, .rem);
6629
6630 return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
6631}6456}
66326457
6633/// Remainder after floor division, defined by:6458fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6634/// @divFloor(a, b) * b + @mod(a, b) = a6459 const zcu = cg.pt.zcu;
6635fn airMod(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6636 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;6460 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
66376461 const dst = try cg.resolveInst(bin_op.lhs);
6638 const pt = cg.pt;6462 const dst_ty = cg.typeOf(bin_op.lhs);
6639 const zcu = pt.zcu;6463 const ptr_elem_ty = dst_ty.childType(zcu);
6640 const ty = cg.typeOfIndex(inst);6464 const src = try cg.resolveInst(bin_op.rhs);
6641 const lhs = try cg.resolveInst(bin_op.lhs);6465 const src_ty = cg.typeOf(bin_op.rhs);
6642 const rhs = try cg.resolveInst(bin_op.rhs);6466 const len = switch (dst_ty.ptrSize(zcu)) {
66436467 .slice => blk: {
6644 const result = result: {6468 const slice_len = try cg.sliceLen(dst);
6645 if (ty.isUnsignedInt(zcu)) {6469 if (ptr_elem_ty.abiSize(zcu) != 1) {
6646 break :result try cg.binOp(lhs, rhs, ty, .rem);6470 try cg.emitWValue(slice_len);
6647 }6471 try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) });
6648 if (ty.isSignedInt(zcu)) {6472 try cg.addTag(.i32_mul);
6649 // The wasm rem instruction gives the remainder after truncating division (rounding towards6473 try cg.addLocal(.local_set, slice_len.local.value);
6650 // 0), equivalent to @rem.
6651 // We make use of the fact that:
6652 // @mod(a, b) = @rem(@rem(a, b) + b, b)
6653 const int_bits = ty.intInfo(zcu).bits;
6654 const wasm_bits = toWasmBits(int_bits) orelse {
6655 return cg.fail("TODO: `@mod` for signed integers larger than 64 bits ({d} bits requested)", .{int_bits});
6656 };
6657
6658 if (wasm_bits > 64) {
6659 return cg.fail("TODO: `@mod` for signed integers larger than 64 bits ({d} bits requested)", .{int_bits});
6660 }6474 }
66616475 break :blk slice_len;
6662 _ = try cg.binOp(lhs, rhs, ty, .rem);6476 },
6663 _ = try cg.binOp(.stack, rhs, ty, .add);6477 .one => @as(WValue, .{
6664 break :result try cg.binOp(.stack, rhs, ty, .rem);6478 .imm32 = @as(u32, @intCast(ptr_elem_ty.arrayLen(zcu) * ptr_elem_ty.childType(zcu).abiSize(zcu))),
6665 }6479 }),
6666 if (ty.isAnyFloat()) {6480 .c, .many => unreachable,
6667 const rem = try cg.binOp(lhs, rhs, ty, .rem);
6668 const add = try cg.binOp(rem, rhs, ty, .add);
6669 break :result try cg.binOp(add, rhs, ty, .rem);
6670 }
6671 return cg.fail("TODO: @mod for {f}", .{ty.fmt(pt)});
6672 };6481 };
6482 const dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty);
6483 const src_ptr = try cg.sliceOrArrayPtr(src, src_ty);
6484 try cg.memcpy(dst_ptr, src_ptr, len);
66736485
6674 return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });6486 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
6675}6487}
66766488
6677fn airSatMul(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {6489fn airRetAddr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6678 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;6490 // TODO: Implement this properly once stack serialization is solved
6491 return cg.finishAir(inst, switch (cg.ptr_size) {
6492 .wasm32 => .{ .imm32 = 0 },
6493 .wasm64 => .{ .imm64 = 0 },
6494 }, &.{});
6495}
66796496
6497fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6498 const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
6499 const operand = try cg.resolveInst(un_op);
6500 // Each entry to this table is a slice (ptr+len).
6501 // The operand in this instruction represents the index within this table.
6502 // This means to get the final name, we emit the base pointer and then perform
6503 // pointer arithmetic to find the pointer to this slice and return that.
6504 //
6505 // As the names are global and the slice elements are constant, we do not have
6506 // to make a copy of the ptr+value but can point towards them directly.
6680 const pt = cg.pt;6507 const pt = cg.pt;
6681 const zcu = pt.zcu;6508 const name_ty = Type.slice_const_u8_sentinel_0;
6682 const ty = cg.typeOfIndex(inst);6509 const abi_size = name_ty.abiSize(pt.zcu);
6683 const int_info = ty.intInfo(zcu);
6684 const is_signed = int_info.signedness == .signed;
6685
6686 const lhs = try cg.resolveInst(bin_op.lhs);
6687 const rhs = try cg.resolveInst(bin_op.rhs);
6688 const wasm_bits = toWasmBits(int_info.bits) orelse {
6689 return cg.fail("TODO: mul_sat for {f}", .{ty.fmt(pt)});
6690 };
6691
6692 switch (wasm_bits) {
6693 32 => {
6694 const upcast_ty: Type = if (is_signed) Type.i64 else Type.u64;
6695 const lhs_up = try cg.intcast(lhs, ty, upcast_ty);
6696 const rhs_up = try cg.intcast(rhs, ty, upcast_ty);
6697 var mul_res = try (try cg.binOp(lhs_up, rhs_up, upcast_ty, .mul)).toLocal(cg, upcast_ty);
6698 defer mul_res.free(cg);
6699 if (is_signed) {
6700 const imm_max: WValue = .{ .imm64 = ~@as(u64, 0) >> @intCast(64 - (int_info.bits - 1)) };
6701 try cg.emitWValue(mul_res);
6702 try cg.emitWValue(imm_max);
6703 _ = try cg.cmp(mul_res, imm_max, upcast_ty, .lt);
6704 try cg.addTag(.select);
6705
6706 var tmp = try cg.allocLocal(upcast_ty);
6707 defer tmp.free(cg);
6708 try cg.addLocal(.local_set, tmp.local.value);
67096510
6710 const imm_min: WValue = .{ .imm64 = ~@as(u64, 0) << @intCast(int_info.bits - 1) };6511 // Lowers to a i32.const or i64.const with the error table memory address.
6711 try cg.emitWValue(tmp);6512 cg.error_name_table_ref_count += 1;
6712 try cg.emitWValue(imm_min);6513 try cg.addTag(.error_name_table_ref);
6713 _ = try cg.cmp(tmp, imm_min, upcast_ty, .gt);6514 try cg.emitWValue(operand);
6714 try cg.addTag(.select);6515 switch (cg.ptr_size) {
6715 } else {6516 .wasm32 => {
6716 const imm_max: WValue = .{ .imm64 = ~@as(u64, 0) >> @intCast(64 - int_info.bits) };6517 try cg.addImm32(@intCast(abi_size));
6717 try cg.emitWValue(mul_res);6518 try cg.addTag(.i32_mul);
6718 try cg.emitWValue(imm_max);6519 try cg.addTag(.i32_add);
6719 _ = try cg.cmp(mul_res, imm_max, upcast_ty, .lt);
6720 try cg.addTag(.select);
6721 }
6722 try cg.addTag(.i32_wrap_i64);
6723 },
6724 64 => {
6725 if (!(int_info.bits == 64 and int_info.signedness == .signed)) {
6726 return cg.fail("TODO: mul_sat for {f}", .{ty.fmt(pt)});
6727 }
6728 const overflow_ret = try cg.allocStack(Type.i32);
6729 _ = try cg.callIntrinsic(
6730 .__mulodi4,
6731 &[_]InternPool.Index{ .i64_type, .i64_type, .usize_type },
6732 Type.i64,
6733 &.{ lhs, rhs, overflow_ret },
6734 );
6735 const xor = try cg.binOp(lhs, rhs, Type.i64, .xor);
6736 const sign_v = try cg.binOp(xor, .{ .imm64 = 63 }, Type.i64, .shr);
6737 _ = try cg.binOp(sign_v, .{ .imm64 = ~@as(u63, 0) }, Type.i64, .xor);
6738 _ = try cg.load(overflow_ret, Type.i32, 0);
6739 try cg.addTag(.i32_eqz);
6740 try cg.addTag(.select);
6741 },6520 },
6742 128 => {6521 .wasm64 => {
6743 if (!(int_info.bits == 128 and int_info.signedness == .signed)) {6522 try cg.addImm64(abi_size);
6744 return cg.fail("TODO: mul_sat for {f}", .{ty.fmt(pt)});6523 try cg.addTag(.i64_mul);
6745 }6524 try cg.addTag(.i64_add);
6746 const overflow_ret = try cg.allocStack(Type.i32);
6747 const ret = try cg.callIntrinsic(
6748 .__muloti4,
6749 &[_]InternPool.Index{ .i128_type, .i128_type, .usize_type },
6750 Type.i128,
6751 &.{ lhs, rhs, overflow_ret },
6752 );
6753 try cg.lowerToStack(ret);
6754 const xor = try cg.binOp(lhs, rhs, Type.i128, .xor);
6755 const sign_v = try cg.binOp(xor, .{ .imm32 = 127 }, Type.i128, .shr);
6756
6757 // xor ~@as(u127, 0)
6758 try cg.emitWValue(sign_v);
6759 const lsb = try cg.load(sign_v, Type.u64, 0);
6760 _ = try cg.binOp(lsb, .{ .imm64 = ~@as(u64, 0) }, Type.u64, .xor);
6761 try cg.store(.stack, .stack, Type.u64, sign_v.offset());
6762 try cg.emitWValue(sign_v);
6763 const msb = try cg.load(sign_v, Type.u64, 8);
6764 _ = try cg.binOp(msb, .{ .imm64 = ~@as(u63, 0) }, Type.u64, .xor);
6765 try cg.store(.stack, .stack, Type.u64, sign_v.offset() + 8);
6766
6767 try cg.lowerToStack(sign_v);
6768 _ = try cg.load(overflow_ret, Type.i32, 0);
6769 try cg.addTag(.i32_eqz);
6770 try cg.addTag(.select);
6771 },6525 },
6772 else => unreachable,
6773 }
6774 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6775}
6776
6777fn airSatBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: Op) InnerError!void {
6778 assert(op == .add or op == .sub);
6779 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6780
6781 const zcu = cg.pt.zcu;
6782 const ty = cg.typeOfIndex(inst);
6783 const lhs = try cg.resolveInst(bin_op.lhs);
6784 const rhs = try cg.resolveInst(bin_op.rhs);
6785
6786 const int_info = ty.intInfo(zcu);
6787 const is_signed = int_info.signedness == .signed;
6788
6789 if (int_info.bits > 64) {
6790 return cg.fail("TODO: saturating arithmetic for integers with bitsize '{d}'", .{int_info.bits});
6791 }6526 }
67926527
6793 if (is_signed) {6528 return cg.finishAir(inst, .stack, &.{un_op});
6794 const result = try signedSat(cg, lhs, rhs, ty, op);6529}
6795 return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
6796 }
6797
6798 const wasm_bits = toWasmBits(int_info.bits).?;
6799 var bin_result = try (try cg.binOp(lhs, rhs, ty, op)).toLocal(cg, ty);
6800 defer bin_result.free(cg);
6801 if (wasm_bits != int_info.bits and op == .add) {
6802 const val: u64 = @as(u64, @intCast((@as(u65, 1) << @as(u7, @intCast(int_info.bits))) - 1));
6803 const imm_val: WValue = switch (wasm_bits) {
6804 32 => .{ .imm32 = @intCast(val) },
6805 64 => .{ .imm64 = val },
6806 else => unreachable,
6807 };
6808
6809 try cg.emitWValue(bin_result);
6810 try cg.emitWValue(imm_val);
6811 _ = try cg.cmp(bin_result, imm_val, ty, .lt);
6812 } else {
6813 switch (wasm_bits) {
6814 32 => try cg.addImm32(if (op == .add) std.math.maxInt(u32) else 0),
6815 64 => try cg.addImm64(if (op == .add) std.math.maxInt(u64) else 0),
6816 else => unreachable,
6817 }
6818 try cg.emitWValue(bin_result);
6819 _ = try cg.cmp(bin_result, lhs, ty, if (op == .add) .lt else .gt);
6820 }
68216530
6822 try cg.addTag(.select);6531fn airPtrSliceFieldPtr(cg: *CodeGen, inst: Air.Inst.Index, offset: u32) InnerError!void {
6823 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });6532 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6533 const slice_ptr = try cg.resolveInst(ty_op.operand);
6534 const result = try cg.buildPointerOffset(slice_ptr, offset, .new);
6535 return cg.finishAir(inst, result, &.{ty_op.operand});
6824}6536}
68256537
6826fn signedSat(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue {6538fn airDbgStmt(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6827 const pt = cg.pt;6539 const dbg_stmt = cg.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;
6828 const zcu = pt.zcu;6540 try cg.addInst(.{ .tag = .dbg_line, .data = .{
6829 const int_info = ty.intInfo(zcu);6541 .payload = try cg.addExtra(Mir.DbgLineColumn{
6830 const wasm_bits = toWasmBits(int_info.bits).?;6542 .line = dbg_stmt.line,
6831 const is_wasm_bits = wasm_bits == int_info.bits;6543 .column = dbg_stmt.column,
6832 const ext_ty = if (!is_wasm_bits) try pt.intType(int_info.signedness, wasm_bits) else ty;6544 }),
68336545 } });
6834 const max_val: u64 = @as(u64, @intCast((@as(u65, 1) << @as(u7, @intCast(int_info.bits - 1))) - 1));6546 return cg.finishAir(inst, .none, &.{});
6835 const min_val: i64 = (-@as(i64, @intCast(@as(u63, @intCast(max_val))))) - 1;6547}
6836 const max_wvalue: WValue = switch (wasm_bits) {
6837 32 => .{ .imm32 = @truncate(max_val) },
6838 64 => .{ .imm64 = max_val },
6839 else => unreachable,
6840 };
6841 const min_wvalue: WValue = switch (wasm_bits) {
6842 32 => .{ .imm32 = @bitCast(@as(i32, @truncate(min_val))) },
6843 64 => .{ .imm64 = @bitCast(min_val) },
6844 else => unreachable,
6845 };
68466548
6847 var bin_result = try (try cg.binOp(lhs, rhs, ext_ty, op)).toLocal(cg, ext_ty);6549fn airDbgInlineBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6848 if (!is_wasm_bits) {6550 const block = cg.air.unwrapDbgBlock(inst);
6849 defer bin_result.free(cg); // not returned in this branch6551 // TODO
6850 try cg.emitWValue(bin_result);6552 try cg.lowerBlock(inst, block.ty, block.body);
6851 try cg.emitWValue(max_wvalue);6553}
6852 _ = try cg.cmp(bin_result, max_wvalue, ext_ty, .lt);
6853 try cg.addTag(.select);
6854 try cg.addLocal(.local_set, bin_result.local.value); // re-use local
68556554
6856 try cg.emitWValue(bin_result);6555fn airDbgVar(
6857 try cg.emitWValue(min_wvalue);6556 cg: *CodeGen,
6858 _ = try cg.cmp(bin_result, min_wvalue, ext_ty, .gt);6557 inst: Air.Inst.Index,
6859 try cg.addTag(.select);6558 local_tag: link.File.Dwarf.WipNav.LocalVarTag,
6860 try cg.addLocal(.local_set, bin_result.local.value); // re-use local6559 is_ptr: bool,
6861 return (try cg.wrapOperand(bin_result, ty)).toLocal(cg, ty);6560) InnerError!void {
6862 } else {6561 _ = is_ptr;
6863 const zero: WValue = switch (wasm_bits) {6562 _ = local_tag;
6864 32 => .{ .imm32 = 0 },6563 return cg.finishAir(inst, .none, &.{});
6865 64 => .{ .imm64 = 0 },
6866 else => unreachable,
6867 };
6868 try cg.emitWValue(max_wvalue);
6869 try cg.emitWValue(min_wvalue);
6870 _ = try cg.cmp(bin_result, zero, ty, .lt);
6871 try cg.addTag(.select);
6872 try cg.emitWValue(bin_result);
6873 // leave on stack
6874 const cmp_zero_result = try cg.cmp(rhs, zero, ty, if (op == .add) .lt else .gt);
6875 const cmp_bin_result = try cg.cmp(bin_result, lhs, ty, .lt);
6876 _ = try cg.binOp(cmp_zero_result, cmp_bin_result, Type.u32, .xor); // comparisons always return i32, so provide u32 as type to xor.
6877 try cg.addTag(.select);
6878 try cg.addLocal(.local_set, bin_result.local.value); // re-use local
6879 return bin_result;
6880 }
6881}6564}
68826565
6883fn airShlSat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {6566fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6884 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;6567 const unwrapped_try = cg.air.unwrapTry(inst);
6568 const body = unwrapped_try.else_body;
6569 const err_union = try cg.resolveInst(unwrapped_try.error_union);
6570 const err_union_ty = cg.typeOf(unwrapped_try.error_union);
6571 const result = try lowerTry(cg, inst, err_union, body, err_union_ty, false);
6572 return cg.finishAir(inst, result, &.{unwrapped_try.error_union});
6573}
68856574
6886 const pt = cg.pt;6575fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6887 const zcu = pt.zcu;6576 const zcu = cg.pt.zcu;
6577 const unwrapped_try = cg.air.unwrapTryPtr(inst);
6578 const err_union_ptr = try cg.resolveInst(unwrapped_try.error_union_ptr);
6579 const body = unwrapped_try.else_body;
6580 const err_union_ty = cg.typeOf(unwrapped_try.error_union_ptr).childType(zcu);
6581 const result = try lowerTry(cg, inst, err_union_ptr, body, err_union_ty, true);
6582 return cg.finishAir(inst, result, &.{unwrapped_try.error_union_ptr});
6583}
68886584
6889 if (cg.typeOf(bin_op.lhs).isVector(zcu) and !cg.typeOf(bin_op.rhs).isVector(zcu)) {6585fn lowerTry(
6890 return cg.fail("TODO: implement vector 'shl_sat' with scalar rhs", .{});6586 cg: *CodeGen,
6891 }6587 inst: Air.Inst.Index,
6588 err_union: WValue,
6589 body: []const Air.Inst.Index,
6590 err_union_ty: Type,
6591 operand_is_ptr: bool,
6592) InnerError!WValue {
6593 const zcu = cg.pt.zcu;
68926594
6893 const ty = cg.typeOfIndex(inst);6595 const pl_ty = err_union_ty.errorUnionPayload(zcu);
6894 const int_info = ty.intInfo(zcu);6596 const pl_has_bits = pl_ty.hasRuntimeBits(zcu);
6895 const is_signed = int_info.signedness == .signed;
6896 if (int_info.bits > 64) {
6897 return cg.fail("TODO: Saturating shifting left for integers with bitsize '{d}'", .{int_info.bits});
6898 }
68996597
6900 const wasm_bits = toWasmBits(int_info.bits).?;6598 if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
6599 // Block we can jump out of when error is not set
6600 try cg.startBlock(.block, .empty);
69016601
6902 const lhs = try cg.resolveInst(bin_op.lhs);6602 // check if the error tag is set for the error union.
6903 const rhs = rhs: {6603 try cg.emitWValue(err_union);
6904 const rhs = try cg.resolveInst(bin_op.rhs);6604 if (pl_has_bits or operand_is_ptr) {
6905 const rhs_ty = cg.typeOf(bin_op.rhs);6605 const err_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
6906 // The type of `rhs` is the log2 int of the type of `lhs`, but WASM wants the lhs and rhs types to match.6606 try cg.addMemArg(.i32_load16_u, .{
6907 if (toWasmBits(@intCast(rhs_ty.bitSize(zcu))).? == wasm_bits) {6607 .offset = err_union.offset() + err_offset,
6908 break :rhs rhs; // the WASM types match, so no cast necessary6608 .alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),
6609 });
6909 }6610 }
6910 const casted = try cg.intcast(rhs, rhs_ty, ty);6611 try cg.addTag(.i32_eqz);
6911 break :rhs try casted.toLocal(cg, ty);6612 try cg.addLabel(.br_if, 0); // jump out of block when error is '0'
6912 };
6913
6914 const result = try cg.allocLocal(ty);
6915
6916 if (wasm_bits == int_info.bits) {
6917 var shl = try (try cg.binOp(lhs, rhs, ty, .shl)).toLocal(cg, ty);
6918 defer shl.free(cg);
6919 var shr = try (try cg.binOp(shl, rhs, ty, .shr)).toLocal(cg, ty);
6920 defer shr.free(cg);
69216613
6922 switch (wasm_bits) {6614 const liveness = cg.liveness.getCondBr(inst);
6923 32 => blk: {6615 try cg.branches.append(cg.gpa, .{});
6924 if (!is_signed) {6616 try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, liveness.else_deaths.len + liveness.then_deaths.len);
6925 try cg.addImm32(std.math.maxInt(u32));6617 defer {
6926 break :blk;6618 var branch = cg.branches.pop().?;
6927 }6619 branch.deinit(cg.gpa);
6928 try cg.addImm32(@bitCast(@as(i32, std.math.minInt(i32))));
6929 try cg.addImm32(@bitCast(@as(i32, std.math.maxInt(i32))));
6930 _ = try cg.cmp(lhs, .{ .imm32 = 0 }, ty, .lt);
6931 try cg.addTag(.select);
6932 },
6933 64 => blk: {
6934 if (!is_signed) {
6935 try cg.addImm64(std.math.maxInt(u64));
6936 break :blk;
6937 }
6938 try cg.addImm64(@bitCast(@as(i64, std.math.minInt(i64))));
6939 try cg.addImm64(@bitCast(@as(i64, std.math.maxInt(i64))));
6940 _ = try cg.cmp(lhs, .{ .imm64 = 0 }, ty, .lt);
6941 try cg.addTag(.select);
6942 },
6943 else => unreachable,
6944 }6620 }
6945 try cg.emitWValue(shl);6621 try cg.genBody(body);
6946 _ = try cg.cmp(lhs, shr, ty, .neq);6622 try cg.endBlock();
6947 try cg.addTag(.select);6623 }
6948 try cg.addLocal(.local_set, result.local.value);
6949 } else {
6950 const shift_size = wasm_bits - int_info.bits;
6951 const shift_value: WValue = switch (wasm_bits) {
6952 32 => .{ .imm32 = shift_size },
6953 64 => .{ .imm64 = shift_size },
6954 else => unreachable,
6955 };
6956 const ext_ty = try pt.intType(int_info.signedness, wasm_bits);
6957
6958 var shl_res = try (try cg.binOp(lhs, shift_value, ext_ty, .shl)).toLocal(cg, ext_ty);
6959 defer shl_res.free(cg);
6960 var shl = try (try cg.binOp(shl_res, rhs, ext_ty, .shl)).toLocal(cg, ext_ty);
6961 defer shl.free(cg);
6962 var shr = try (try cg.binOp(shl, rhs, ext_ty, .shr)).toLocal(cg, ext_ty);
6963 defer shr.free(cg);
6964
6965 switch (wasm_bits) {
6966 32 => blk: {
6967 if (!is_signed) {
6968 try cg.addImm32(std.math.maxInt(u32));
6969 break :blk;
6970 }
6971
6972 try cg.addImm32(@bitCast(@as(i32, std.math.minInt(i32))));
6973 try cg.addImm32(@bitCast(@as(i32, std.math.maxInt(i32))));
6974 _ = try cg.cmp(shl_res, .{ .imm32 = 0 }, ext_ty, .lt);
6975 try cg.addTag(.select);
6976 },
6977 64 => blk: {
6978 if (!is_signed) {
6979 try cg.addImm64(std.math.maxInt(u64));
6980 break :blk;
6981 }
69826624
6983 try cg.addImm64(@bitCast(@as(i64, std.math.minInt(i64))));6625 // if we reach here it means error was not set, and we want the payload
6984 try cg.addImm64(@bitCast(@as(i64, std.math.maxInt(i64))));6626 if (!pl_has_bits and !operand_is_ptr) {
6985 _ = try cg.cmp(shl_res, .{ .imm64 = 0 }, ext_ty, .lt);6627 return .none;
6986 try cg.addTag(.select);
6987 },
6988 else => unreachable,
6989 }
6990 try cg.emitWValue(shl);
6991 _ = try cg.cmp(shl_res, shr, ext_ty, .neq);
6992 try cg.addTag(.select);
6993 try cg.addLocal(.local_set, result.local.value);
6994 var shift_result = try cg.binOp(result, shift_value, ext_ty, .shr);
6995 if (is_signed) {
6996 shift_result = try cg.wrapOperand(shift_result, ty);
6997 }
6998 try cg.addLocal(.local_set, result.local.value);
6999 }6628 }
70006629
7001 return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });6630 const pl_offset: u32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));
6631 if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) {
6632 return buildPointerOffset(cg, err_union, pl_offset, .new);
6633 }
6634 const payload = try cg.load(err_union, pl_ty, pl_offset);
6635 return payload.toLocal(cg, pl_ty);
7002}6636}
70036637
7004/// Calls a compiler-rt intrinsic by creating an undefined symbol,6638/// Calls a compiler-rt intrinsic by creating an undefined symbol,
...@@ -7154,6 +6788,8 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7154,6 +6788,8 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7154 const ty = ptr_ty.childType(zcu);6788 const ty = ptr_ty.childType(zcu);
7155 const result_ty = cg.typeOfIndex(inst);6789 const result_ty = cg.typeOfIndex(inst);
71566790
6791 const int_ty: IntType = .fromType(cg, ty);
6792
7157 const ptr_operand = try cg.resolveInst(extra.ptr);6793 const ptr_operand = try cg.resolveInst(extra.ptr);
7158 const expected_val = try cg.resolveInst(extra.expected_value);6794 const expected_val = try cg.resolveInst(extra.expected_value);
7159 const new_val = try cg.resolveInst(extra.new_value);6795 const new_val = try cg.resolveInst(extra.new_value);
...@@ -7176,7 +6812,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7176,7 +6812,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7176 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),6812 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
7177 });6813 });
7178 try cg.addLocal(.local_tee, val_local.local.value);6814 try cg.addLocal(.local_tee, val_local.local.value);
7179 _ = try cg.cmp(.stack, expected_val, ty, .eq);6815 _ = try cg.intCmp(int_ty, .eq, .stack, expected_val);
7180 try cg.addLocal(.local_set, cmp_result.local.value);6816 try cg.addLocal(.local_set, cmp_result.local.value);
7181 break :val val_local;6817 break :val val_local;
7182 } else val: {6818 } else val: {
...@@ -7188,7 +6824,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7188,7 +6824,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7188 try cg.lowerToStack(ptr_operand);6824 try cg.lowerToStack(ptr_operand);
7189 try cg.lowerToStack(new_val);6825 try cg.lowerToStack(new_val);
7190 try cg.emitWValue(ptr_val);6826 try cg.emitWValue(ptr_val);
7191 _ = try cg.cmp(ptr_val, expected_val, ty, .eq);6827 _ = try cg.intCmp(int_ty, .eq, ptr_val, expected_val);
7192 try cg.addLocal(.local_tee, cmp_result.local.value);6828 try cg.addLocal(.local_tee, cmp_result.local.value);
7193 try cg.addTag(.select);6829 try cg.addTag(.select);
7194 try cg.store(.stack, .stack, ty, 0);6830 try cg.store(.stack, .stack, ty, 0);
...@@ -7254,6 +6890,8 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7254,6 +6890,8 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7254 const ty = cg.typeOfIndex(inst);6890 const ty = cg.typeOfIndex(inst);
7255 const op: std.builtin.AtomicRmwOp = extra.op();6891 const op: std.builtin.AtomicRmwOp = extra.op();
72566892
6893 const int_ty: IntType = .fromType(cg, ty);
6894
7257 if (cg.useAtomicFeature()) {6895 if (cg.useAtomicFeature()) {
7258 switch (op) {6896 switch (op) {
7259 .Max,6897 .Max,
...@@ -7269,20 +6907,19 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7269,20 +6907,19 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7269 try cg.emitWValue(ptr);6907 try cg.emitWValue(ptr);
7270 try cg.emitWValue(value);6908 try cg.emitWValue(value);
7271 if (op == .Nand) {6909 if (op == .Nand) {
7272 const wasm_bits = toWasmBits(@intCast(ty.bitSize(zcu))).?;6910 const and_res = try cg.intAnd(int_ty, value, operand);
72736911 if (int_ty.bits <= 32) {
7274 const and_res = try cg.binOp(value, operand, ty, .@"and");6912 try cg.addImm32(~@as(u32, 0));
7275 if (wasm_bits == 32)6913 } else if (int_ty.bits <= 64) {
7276 try cg.addImm32(~@as(u32, 0))6914 try cg.addImm64(~@as(u64, 0));
7277 else if (wasm_bits == 64)6915 } else {
7278 try cg.addImm64(~@as(u64, 0))
7279 else
7280 return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});6916 return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});
7281 _ = try cg.binOp(and_res, .stack, ty, .xor);6917 }
6918 _ = try cg.intXor(int_ty, and_res, .stack);
7282 } else {6919 } else {
7283 try cg.emitWValue(value);6920 try cg.emitWValue(value);
7284 try cg.emitWValue(operand);6921 try cg.emitWValue(operand);
7285 _ = try cg.cmp(value, operand, ty, if (op == .Max) .gt else .lt);6922 _ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, value, operand);
7286 try cg.addTag(.select);6923 try cg.addTag(.select);
7287 }6924 }
7288 try cg.addAtomicMemArg(6925 try cg.addAtomicMemArg(
...@@ -7300,7 +6937,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7300,7 +6937,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7300 );6937 );
7301 const select_res = try cg.allocLocal(ty);6938 const select_res = try cg.allocLocal(ty);
7302 try cg.addLocal(.local_tee, select_res.local.value);6939 try cg.addLocal(.local_tee, select_res.local.value);
7303 _ = try cg.cmp(.stack, value, ty, .neq); // leave on stack so we can use it for br_if6940 _ = try cg.intCmp(int_ty, .neq, .stack, value); // leave on stack so we can use it for br_if
73046941
7305 try cg.emitWValue(select_res);6942 try cg.emitWValue(select_res);
7306 try cg.addLocal(.local_set, value.local.value);6943 try cg.addLocal(.local_set, value.local.value);
...@@ -7375,16 +7012,16 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7375,16 +7012,16 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7375 .Xor,7012 .Xor,
7376 => {7013 => {
7377 try cg.emitWValue(ptr);7014 try cg.emitWValue(ptr);
7378 _ = try cg.binOp(result, operand, ty, switch (op) {7015 _ = switch (op) {
7379 .Add => .add,7016 .Add => try cg.intAdd(int_ty, result, operand),
7380 .Sub => .sub,7017 .Sub => try cg.intSub(int_ty, result, operand),
7381 .And => .@"and",7018 .And => try cg.intAnd(int_ty, result, operand),
7382 .Or => .@"or",7019 .Or => try cg.intOr(int_ty, result, operand),
7383 .Xor => .xor,7020 .Xor => try cg.intXor(int_ty, result, operand),
7384 else => unreachable,7021 else => unreachable,
7385 });7022 };
7386 if (ty.isInt(zcu) and (op == .Add or op == .Sub)) {7023 if (ty.isInt(zcu) and (op == .Add or op == .Sub)) {
7387 _ = try cg.wrapOperand(.stack, ty);7024 _ = try cg.intWrap(int_ty, .stack);
7388 }7025 }
7389 try cg.store(.stack, .stack, ty, ptr.offset());7026 try cg.store(.stack, .stack, ty, ptr.offset());
7390 },7027 },
...@@ -7394,22 +7031,21 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7394,22 +7031,21 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7394 try cg.emitWValue(ptr);7031 try cg.emitWValue(ptr);
7395 try cg.emitWValue(result);7032 try cg.emitWValue(result);
7396 try cg.emitWValue(operand);7033 try cg.emitWValue(operand);
7397 _ = try cg.cmp(result, operand, ty, if (op == .Max) .gt else .lt);7034 _ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, result, operand);
7398 try cg.addTag(.select);7035 try cg.addTag(.select);
7399 try cg.store(.stack, .stack, ty, ptr.offset());7036 try cg.store(.stack, .stack, ty, ptr.offset());
7400 },7037 },
7401 .Nand => {7038 .Nand => {
7402 const wasm_bits = toWasmBits(@intCast(ty.bitSize(zcu))).?;
7403
7404 try cg.emitWValue(ptr);7039 try cg.emitWValue(ptr);
7405 const and_res = try cg.binOp(result, operand, ty, .@"and");7040 const and_res = try cg.intAnd(int_ty, result, operand);
7406 if (wasm_bits == 32)7041 if (int_ty.bits <= 32) {
7407 try cg.addImm32(~@as(u32, 0))7042 try cg.addImm32(~@as(u32, 0));
7408 else if (wasm_bits == 64)7043 } else if (int_ty.bits <= 64) {
7409 try cg.addImm64(~@as(u64, 0))7044 try cg.addImm64(~@as(u64, 0));
7410 else7045 } else {
7411 return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});7046 return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});
7412 _ = try cg.binOp(and_res, .stack, ty, .xor);7047 }
7048 _ = try cg.intXor(int_ty, and_res, .stack);
7413 try cg.store(.stack, .stack, ty, ptr.offset());7049 try cg.store(.stack, .stack, ty, ptr.offset());
7414 },7050 },
7415 }7051 }
...@@ -7478,38 +7114,3 @@ fn typeOfIndex(cg: *CodeGen, inst: Air.Inst.Index) Type {...@@ -7478,38 +7114,3 @@ fn typeOfIndex(cg: *CodeGen, inst: Air.Inst.Index) Type {
7478 const zcu = cg.pt.zcu;7114 const zcu = cg.pt.zcu;
7479 return cg.air.typeOfIndex(inst, &zcu.intern_pool);7115 return cg.air.typeOfIndex(inst, &zcu.intern_pool);
7480}7116}
7481
7482fn floatCmpIntrinsic(op: std.math.CompareOperator, bits: u16) Mir.Intrinsic {
7483 return switch (op) {
7484 .lt => switch (bits) {
7485 80 => .__ltxf2,
7486 128 => .__lttf2,
7487 else => unreachable,
7488 },
7489 .lte => switch (bits) {
7490 80 => .__lexf2,
7491 128 => .__letf2,
7492 else => unreachable,
7493 },
7494 .eq => switch (bits) {
7495 80 => .__eqxf2,
7496 128 => .__eqtf2,
7497 else => unreachable,
7498 },
7499 .neq => switch (bits) {
7500 80 => .__nexf2,
7501 128 => .__netf2,
7502 else => unreachable,
7503 },
7504 .gte => switch (bits) {
7505 80 => .__gexf2,
7506 128 => .__getf2,
7507 else => unreachable,
7508 },
7509 .gt => switch (bits) {
7510 80 => .__gtxf2,
7511 128 => .__gttf2,
7512 else => unreachable,
7513 },
7514 };
7515}
test/behavior/atomics.zig+1
...@@ -189,6 +189,7 @@ test "atomicrmw with floats" {...@@ -189,6 +189,7 @@ test "atomicrmw with floats" {
189 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO189 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
190 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;190 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
191 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO191 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
192 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
192193
193 try testAtomicRmwFloat();194 try testAtomicRmwFloat();
194 try comptime testAtomicRmwFloat();195 try comptime testAtomicRmwFloat();
test/behavior/field_parent_ptr.zig-5
...@@ -586,7 +586,6 @@ test "@fieldParentPtr extern struct last zero-bit field" {...@@ -586,7 +586,6 @@ test "@fieldParentPtr extern struct last zero-bit field" {
586}586}
587587
588test "@fieldParentPtr unaligned packed struct" {588test "@fieldParentPtr unaligned packed struct" {
589 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
590 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;589 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
591 if (builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;590 if (builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
592 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;591 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
...@@ -726,7 +725,6 @@ test "@fieldParentPtr unaligned packed struct" {...@@ -726,7 +725,6 @@ test "@fieldParentPtr unaligned packed struct" {
726}725}
727726
728test "@fieldParentPtr aligned packed struct" {727test "@fieldParentPtr aligned packed struct" {
729 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
730 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;728 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
731 if (builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;729 if (builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
732 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;730 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
...@@ -1033,7 +1031,6 @@ test "@fieldParentPtr packed struct first zero-bit field" {...@@ -1033,7 +1031,6 @@ test "@fieldParentPtr packed struct first zero-bit field" {
1033 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;1031 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
1034 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;1032 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1035 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;1033 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
1036 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
10371034
1038 const C = packed struct {1035 const C = packed struct {
1039 a: u0 = 0,1036 a: u0 = 0,
...@@ -1140,7 +1137,6 @@ test "@fieldParentPtr packed struct middle zero-bit field" {...@@ -1140,7 +1137,6 @@ test "@fieldParentPtr packed struct middle zero-bit field" {
1140 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;1137 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
1141 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;1138 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1142 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;1139 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
1143 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
11441140
1145 const C = packed struct {1141 const C = packed struct {
1146 a: f32 = 3.14,1142 a: f32 = 3.14,
...@@ -1247,7 +1243,6 @@ test "@fieldParentPtr packed struct last zero-bit field" {...@@ -1247,7 +1243,6 @@ test "@fieldParentPtr packed struct last zero-bit field" {
1247 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;1243 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
1248 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;1244 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1249 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;1245 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
1250 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
12511246
1252 const C = packed struct {1247 const C = packed struct {
1253 a: f32 = 3.14,1248 a: f32 = 3.14,
test/behavior/struct.zig-1
...@@ -749,7 +749,6 @@ test "packed struct with fp fields" {...@@ -749,7 +749,6 @@ test "packed struct with fp fields" {
749 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;749 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
750 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;750 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
751 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;751 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
752 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
753752
754 const S = packed struct {753 const S = packed struct {
755 data0: f32,754 data0: f32,