1//! Machine Intermediate Representation.
2//! This data is produced by x86_64 Codegen and consumed by x86_64 Isel.
3//! These instructions have a 1:1 correspondence with machine code instructions
4//! for the target. MIR can be lowered to source-annotated textual assembly code
5//! instructions, or it can be lowered to machine code.
6//! The main purpose of MIR is to postpone the assignment of offsets until Isel,
7//! so that, for example, the smaller encodings of jump instructions can be used.
8
9instructions: std.MultiArrayList(Inst).Slice,
10/// The meaning of this data is determined by `Inst.Tag` value.
11extra: []const u32,
12string_bytes: []const u8,
13locals: []const Local,
14table: []const Inst.Index,
15/// Optional data which, when present, can be used to accelerate encoding speed.
16memoized_encodings: []const u0 = &.{},
17frame_locs: std.MultiArrayList(FrameLoc).Slice,
18
19pub const Inst = struct {
20 tag: Tag,
21 ops: Ops,
22 data: Data,
23
24 pub const Index = u32;
25
26 pub const Fixes = enum(u8) {
27 /// ___
28 @"_",
29
30 /// ___ 0
31 _0,
32 /// ___ 1
33 _1,
34 /// ___ 2
35 _2,
36 /// ___ 3
37 _3,
38 /// ___ 4
39 _4,
40
41 /// ___ Demote
42 _demote,
43 /// ___ Flush
44 _flush,
45 /// ___ Flush Optimized
46 _flushopt,
47 /// ___ Instructions With T0 Hint
48 _it0,
49 /// ___ Instructions With T0 Hint
50 _it1,
51 /// ___ With NTA Hint
52 _nta,
53 /// System Call ___
54 sys_,
55 /// ___ With T0 Hint
56 _t0,
57 /// ___ With T1 Hint
58 _t1,
59 /// ___ With T2 Hint
60 _t2,
61 /// ___ Write Back
62 _wb,
63 /// ___ With Intent to Write and T1 Hint
64 _wt1,
65
66 /// ___ crement Shadow Stack Pointer Doubleword
67 _csspd,
68 /// ___ crement Shadow Stack Pointer Quadword
69 _csspq,
70 /// ___ FS Segment Base
71 _fsbase,
72 /// ___ GS
73 _gs,
74 /// ___ GS Segment Base
75 _gsbase,
76 /// ___ Model Specific Register
77 _msr,
78 /// ___ MXCSR
79 _mxcsr,
80 /// ___ Processor ID
81 _pid,
82 /// ___ Protection Key Rights For User Pages
83 _pkru,
84 /// ___ Performance-Monitoring Counters
85 _pmc,
86 /// ___ Random Number
87 _rand,
88 /// ___ r Busy Flag in a Supervisor Shadow Stack token
89 _rssbsy,
90 /// ___ Random Seed
91 _seed,
92 /// ___ Shadow Stack Doubleword
93 _ssd,
94 /// ___ Shadow Stack Quadword
95 _ssq,
96 /// ___ Shadow Stack Pointer Doubleword
97 _sspd,
98 /// ___ Shadow Stack Pointer Quadword
99 _sspq,
100 /// ___ Time-Stamp Counter
101 _tsc,
102 /// ___ Time-Stamp Counter And Processor ID
103 _tscp,
104 /// ___ User Shadow Stack Doubleword
105 _ussd,
106 /// ___ User Shadow Stack Quadword
107 _ussq,
108 /// VEX-Encoded ___ MXCSR
109 v_mxcsr,
110
111 /// Byte ___
112 b_,
113 /// Interrupt ___
114 /// Integer ___
115 i_,
116 /// Interrupt ___ Word
117 i_w,
118 /// Interrupt ___ Doubleword
119 i_d,
120 /// Interrupt ___ Quadword
121 i_q,
122 /// User-Interrupt ___
123 ui_,
124
125 /// ___ mp
126 _mp,
127 /// ___ if CX register is 0
128 _cxz,
129 /// ___ if ECX register is 0
130 _ecxz,
131 /// ___ if RCX register is 0
132 _rcxz,
133
134 /// ___ Addition
135 _a,
136 /// ___ Subtraction
137 _s,
138 /// ___ Multiply
139 _m,
140 /// ___ Division
141 _d,
142
143 /// ___ Without Affecting Flags
144 _x,
145 /// ___ Left
146 _l,
147 /// ___ Left Double
148 _ld,
149 /// ___ Left Without Affecting Flags
150 _lx,
151 /// ___ Mask
152 _msk,
153 /// ___ Right
154 /// ___ For Reading
155 /// ___ Register
156 _r,
157 /// ___ Right Double
158 _rd,
159 /// ___ Right Without Affecting Flags
160 _rx,
161
162 /// ___ Forward
163 _f,
164 /// ___ Reverse
165 //_r,
166
167 /// ___ Above
168 //_a,
169 /// ___ Above Or Equal
170 _ae,
171 /// ___ Below
172 _b,
173 /// ___ Below Or Equal
174 /// ___ Big Endian
175 _be,
176 /// ___ Carry
177 /// ___ Carry Flag
178 _c,
179 /// ___ Equal
180 _e,
181 /// ___ Greater
182 _g,
183 /// ___ Greater Or Equal
184 _ge,
185 /// ___ Less
186 //_l,
187 /// ___ Less Or Equal
188 _le,
189 /// ___ Not Above
190 _na,
191 /// ___ Not Above Or Equal
192 _nae,
193 /// ___ Not Below
194 _nb,
195 /// ___ Not Below Or Equal
196 _nbe,
197 /// ___ Not Carry
198 _nc,
199 /// ___ Not Equal
200 _ne,
201 /// ___ Not Greater
202 _ng,
203 /// ___ Not Greater Or Equal
204 _nge,
205 /// ___ Not Less
206 _nl,
207 /// ___ Not Less Or Equal
208 _nle,
209 /// ___ Not Overflow
210 _no,
211 /// ___ Not Parity
212 _np,
213 /// ___ Not Sign
214 _ns,
215 /// ___ Not Zero
216 _nz,
217 /// ___ Overflow
218 _o,
219 /// ___ Parity
220 _p,
221 /// ___ Parity Even
222 _pe,
223 /// ___ Parity Odd
224 _po,
225 /// ___ Sign
226 //_s,
227 /// ___ Zero
228 _z,
229 /// ___ Alignment Check Flag
230 _ac,
231 /// ___ Direction Flag
232 //_d,
233 /// ___ Interrupt Flag
234 _i,
235 /// ___ Task-Switched Flag In CR0
236 _ts,
237 /// ___ User Interrupt Flag
238 _ui,
239
240 /// ___ Byte
241 //_b,
242 /// ___ Word
243 /// ___ For Writing
244 /// ___ With Intent to Write
245 _w,
246 /// ___ Doubleword
247 //_d,
248 /// ___ Double Quadword to Quadword
249 _dq2q,
250 /// ___ QuadWord
251 _q,
252 /// ___ Quadword to Double Quadword
253 _q2dq,
254
255 /// ___ String
256 //_s,
257 /// ___ String Byte
258 _sb,
259 /// ___ String Word
260 _sw,
261 /// ___ String Doubleword
262 _sd,
263 /// ___ String Quadword
264 _sq,
265
266 /// Repeat ___ String
267 @"rep _s",
268 /// Repeat ___ String Byte
269 @"rep _sb",
270 /// Repeat ___ String Word
271 @"rep _sw",
272 /// Repeat ___ String Doubleword
273 @"rep _sd",
274 /// Repeat ___ String Quadword
275 @"rep _sq",
276
277 /// Repeat Equal ___ String
278 @"repe _s",
279 /// Repeat Equal ___ String Byte
280 @"repe _sb",
281 /// Repeat Equal ___ String Word
282 @"repe _sw",
283 /// Repeat Equal ___ String Doubleword
284 @"repe _sd",
285 /// Repeat Equal ___ String Quadword
286 @"repe _sq",
287
288 /// Repeat Not Equal ___ String
289 @"repne _s",
290 /// Repeat Not Equal ___ String Byte
291 @"repne _sb",
292 /// Repeat Not Equal ___ String Word
293 @"repne _sw",
294 /// Repeat Not Equal ___ String Doubleword
295 @"repne _sd",
296 /// Repeat Not Equal ___ String Quadword
297 @"repne _sq",
298
299 /// Repeat Not Zero ___ String
300 @"repnz _s",
301 /// Repeat Not Zero ___ String Byte
302 @"repnz _sb",
303 /// Repeat Not Zero ___ String Word
304 @"repnz _sw",
305 /// Repeat Not Zero ___ String Doubleword
306 @"repnz _sd",
307 /// Repeat Not Zero ___ String Quadword
308 @"repnz _sq",
309
310 /// Repeat Zero ___ String
311 @"repz _s",
312 /// Repeat Zero ___ String Byte
313 @"repz _sb",
314 /// Repeat Zero ___ String Word
315 @"repz _sw",
316 /// Repeat Zero ___ String Doubleword
317 @"repz _sd",
318 /// Repeat Zero ___ String Quadword
319 @"repz _sq",
320
321 /// Locked ___
322 @"lock _",
323 /// ___ And Complement
324 //_c,
325 /// Locked ___ And Complement
326 @"lock _c",
327 /// ___ And Reset
328 //_r,
329 /// Locked ___ And Reset
330 @"lock _r",
331 /// ___ And Set
332 //_s,
333 /// Locked ___ And Set
334 @"lock _s",
335 /// ___ 8 Bytes
336 _8b,
337 /// Locked ___ 8 Bytes
338 @"lock _8b",
339 /// ___ 16 Bytes
340 _16b,
341 /// Locked ___ 16 Bytes
342 @"lock _16b",
343
344 /// Float ___
345 f_,
346 /// Float ___ +1.0
347 /// Float ___ 1
348 f_1,
349 /// Float ___ Below
350 f_b,
351 /// Float ___ Below Or Equal
352 f_be,
353 /// Float ___ Control Word
354 f_cw,
355 /// Float ___ Equal
356 f_e,
357 /// Float ___ Environment
358 f_env,
359 /// Float ___ log_2(e)
360 f_l2e,
361 /// Float ___ log_2(10)
362 f_l2t,
363 /// Float ___ log_10(2)
364 f_lg2,
365 /// Float ___ log_e(2)
366 f_ln2,
367 /// Float ___ Not Below
368 f_nb,
369 /// Float ___ Not Below Or Equal
370 f_nbe,
371 /// Float ___ Not Equal
372 f_ne,
373 /// Float ___ Not Unordered
374 f_nu,
375 /// Float ___ Pop
376 f_p,
377 /// Float ___ +1
378 f_p1,
379 /// Float ___ π
380 f_pi,
381 /// Float ___ Pop Pop
382 f_pp,
383 /// Float ___ crement Stack-Top Pointer
384 f_cstp,
385 /// Float ___ Status Word
386 f_sw,
387 /// Float ___ Unordered
388 f_u,
389 /// Float ___ +0.0
390 f_z,
391 /// Float BCD ___
392 fb_,
393 /// Float BCD ___ Pop
394 fb_p,
395 /// Float And Integer ___
396 fi_,
397 /// Float And Integer ___ Pop
398 fi_p,
399 /// Float No Wait ___
400 fn_,
401 /// Float No Wait ___ Control Word
402 fn_cw,
403 /// Float No Wait ___ Environment
404 fn_env,
405 /// Float No Wait ___ status word
406 fn_sw,
407 /// Float Extended ___
408 fx_,
409 /// Float Extended ___ 64
410 fx_64,
411
412 /// ___ in 32-bit and Compatibility Mode
413 _32,
414 /// ___ in 64-bit Mode
415 _64,
416
417 /// Packed ___
418 p_,
419 /// Packed ___ Byte
420 p_b,
421 /// Packed ___ Word
422 p_w,
423 /// Packed ___ Doubleword
424 p_d,
425 /// Packed ___ Quadword
426 p_q,
427 /// Packed ___ Double Quadword
428 /// Packed ___ Doubleword to Quadword
429 p_dq,
430 /// Packed ___ Unsigned Doubleword to Quadword
431 p_udq,
432 /// Packed Carry-Less ___ Quadword to Double Quadword
433 pcl_qdq,
434 /// Packed Half ___ Doubleword
435 ph_d,
436 /// Packed Half ___ Saturate Word
437 ph_sw,
438 /// Packed Half ___ Word
439 ph_w,
440 /// ___ Aligned Packed Integer Values
441 _dqa,
442 /// ___ Unaligned Packed Integer Values
443 _dqu,
444
445 /// ___ Scalar Single-Precision Values
446 _ss,
447 /// ___ Packed Single-Precision Values
448 _ps,
449 /// ___ Scalar Double-Precision Values
450 //_sd,
451 /// ___ Packed Double-Precision Values
452 _pd,
453 /// Half ___ Packed Single-Precision Values
454 h_ps,
455 /// Half ___ Packed Double-Precision Values
456 h_pd,
457
458 /// ___ Internal Caches
459 //_d,
460 /// ___ TLB Entries
461 _lpg,
462 /// ___ Process-Context Identifier
463 _pcid,
464
465 /// Load ___
466 l_,
467 /// Memory ___
468 m_,
469 /// Store ___
470 s_,
471 /// Timed ___
472 t_,
473 /// User Level Monitor ___
474 um_,
475
476 /// VEX-Encoded ___
477 v_,
478 /// VEX-Encoded ___ Byte
479 v_b,
480 /// VEX-Encoded ___ Word
481 v_w,
482 /// VEX-Encoded ___ Doubleword
483 v_d,
484 /// VEX-Encoded ___ Quadword
485 v_q,
486 /// VEX-Encoded ___ Aligned Packed Integer Values
487 v_dqa,
488 /// VEX-Encoded ___ Unaligned Packed Integer Values
489 v_dqu,
490 /// VEX-Encoded ___ Integer Data
491 v_i128,
492 /// VEX-Encoded Packed ___
493 vp_,
494 /// VEX-Encoded Packed ___ Byte
495 vp_b,
496 /// VEX-Encoded Packed ___ Word
497 vp_w,
498 /// VEX-Encoded Packed ___ Doubleword
499 vp_d,
500 /// VEX-Encoded Packed ___ Quadword
501 vp_q,
502 /// VEX-Encoded Packed ___ Double Quadword
503 /// VEX-Encoded Packed ___ Doubleword to Quadword
504 vp_dq,
505 /// VEX-Encoded Packed ___ Unsigned Doubleword to Quadword
506 vp_udq,
507 /// VEx-Encoded Packed Carry-Less ___ Quadword to Double Quadword
508 vpcl_qdq,
509 /// VEX-Encoded Packed Half ___ Doubleword
510 vph_d,
511 /// VEX-Encoded Packed Half ___ Saturate Word
512 vph_sw,
513 /// VEX-Encoded Packed Half ___ Word
514 vph_w,
515 /// VEX-Encoded ___ Scalar Single-Precision Values
516 v_ss,
517 /// VEX-Encoded ___ Packed Single-Precision Values
518 v_ps,
519 /// VEX-Encoded ___ Scalar Double-Precision Values
520 v_sd,
521 /// VEX-Encoded ___ Packed Double-Precision Values
522 v_pd,
523 /// VEX-Encoded ___ 128-Bits Of Floating-Point Data
524 v_f128,
525 /// VEX-Encoded Half ___ Packed Single-Precision Values
526 vh_ps,
527 /// VEX-Encoded Half ___ Packed Double-Precision Values
528 vh_pd,
529
530 /// ___ 128-bit key with key locker
531 _128,
532 /// ___ 256-bit key with key locker
533 _256,
534 /// ___ with key locker using 128-bit key
535 _128kl,
536 /// ___ with key locker using 256-bit key
537 _256kl,
538 /// ___ with key locker on 8 blocks using 128-bit key
539 _wide128kl,
540 /// ___ with key locker on 8 blocks using 256-bit key
541 _wide256kl,
542
543 /// Mask ___ Byte
544 k_b,
545 /// Mask ___ Word
546 k_w,
547 /// Mask ___ Doubleword
548 k_d,
549 /// Mask ___ Quadword
550 k_q,
551
552 pub fn fromCond(cc: bits.Condition) Fixes {
553 return switch (cc) {
554 inline else => |cc_tag| @field(Fixes, "_" ++ @tagName(cc_tag)),
555 .z_and_np, .nz_or_p => unreachable,
556 };
557 }
558 };
559
560 pub const Tag = enum(u8) {
561 // General-purpose
562 /// ASCII adjust al after addition
563 /// ASCII adjust ax before division
564 /// ASCII adjust ax after multiply
565 /// ASCII adjust al after subtraction
566 aa,
567 /// Add with carry
568 /// Unsigned integer addition of two operands with carry flag
569 adc,
570 /// Add
571 /// Add packed integers
572 /// Add packed single-precision floating-point values
573 /// Add scalar single-precision floating-point values
574 /// Add packed double-precision floating-point values
575 /// Add scalar double-precision floating-point values
576 /// Packed single-precision floating-point horizontal add
577 /// Packed double-precision floating-point horizontal add
578 /// Packed horizontal add
579 /// Packed horizontal add and saturate
580 add,
581 /// Logical and
582 /// Bitwise logical and of packed single-precision floating-point values
583 /// Bitwise logical and of packed double-precision floating-point values
584 @"and",
585 /// Adjust RPL field of segment selector
586 arpl,
587 /// Bit scan forward
588 /// Bit scan reverse
589 bs,
590 /// Byte swap
591 /// Swap GS base register
592 swap,
593 /// Bit test
594 /// Bit test and complement
595 /// Bit test and reset
596 /// Bit test and set
597 bt,
598 /// Check array index against bounds
599 bound,
600 /// Call
601 /// Fast system call
602 call,
603 /// Convert byte to word
604 cbw,
605 /// Convert doubleword to quadword
606 cdq,
607 /// Convert doubleword to quadword
608 cdqe,
609 /// Clear AC flag in EFLAGS register
610 /// Clear carry flag
611 /// Clear direction flag
612 /// Clear interrupt flag
613 /// Clear task-switched flag in CR0
614 /// Clear user interrupt flag
615 /// Cache line demote
616 /// Flush cache line
617 /// Flush cache line optimized
618 /// Clear busy flag in a supervisor shadow stack token
619 /// Cache line write back
620 cl,
621 /// Complement carry flag
622 cmc,
623 /// Conditional move
624 cmov,
625 /// Logical compare
626 /// Compare string
627 /// Compare scalar single-precision floating-point values
628 /// Compare scalar double-precision floating-point values
629 cmp,
630 /// Compare and exchange
631 /// Compare and exchange bytes
632 cmpxchg,
633 /// CPU identification
634 cpuid,
635 /// Convert doubleword to quadword
636 cqo,
637 /// Convert word to doubleword
638 cwd,
639 /// Convert word to doubleword
640 cwde,
641 /// Decimal adjust AL after addition
642 /// Decimal adjust AL after subtraction
643 da,
644 /// Decrement by 1
645 /// Decrement stack-top pointer
646 /// Decrement shadow stack pointer
647 de,
648 /// Unsigned division
649 /// Signed division
650 /// Divide
651 /// Divide packed single-precision floating-point values
652 /// Divide scalar single-precision floating-point values
653 /// Divide packed double-precision floating-point values
654 /// Divide scalar double-precision floating-point values
655 div,
656 /// Terminate and indirect branch in 32-bit and compatibility mode
657 /// Terminate and indirect branch in 64-bit mode
658 endbr,
659 /// Enqueue command
660 /// Enqueue command supervisor
661 enqcmd,
662 /// Make stack frame for procedure parameters
663 /// Fast system call
664 enter,
665 /// Fast return from fast system call
666 exit,
667 /// Load fence
668 /// Memory fence
669 /// Store fence
670 fence,
671 /// Halt
672 hlt,
673 /// History reset
674 hreset,
675 /// Input from port
676 /// Input from port to string
677 /// Increment by 1
678 /// Increment stack-top pointer
679 /// Increment shadow stack pointer
680 in,
681 /// Call to interrupt procedure
682 int,
683 /// Invalidate internal caches
684 /// Invalidate TLB entries
685 /// Invalidate process-context identifier
686 inv,
687 /// Conditional jump
688 /// Jump
689 j,
690 /// Load status flags into AH register
691 lahf,
692 /// Load access right byte
693 lar,
694 /// Load effective address
695 lea,
696 /// High level procedure exit
697 leave,
698 /// Load global descriptor table register
699 lgdt,
700 /// Load interrupt descriptor table register
701 lidt,
702 /// Load local descriptor table register
703 lldt,
704 /// Load machine status word
705 lmsw,
706 /// Load string
707 lod,
708 /// Loop according to ECX counter
709 loop,
710 /// Load segment limit
711 lsl,
712 /// Load task register
713 ltr,
714 /// Count the number of leading zero bits
715 lzcnt,
716 /// Move
717 /// Move data from string to string
718 /// Move data after swapping bytes
719 /// Move scalar single-precision floating-point value
720 /// Move scalar double-precision floating-point value
721 /// Move doubleword
722 /// Move quadword
723 /// Move aligned packed integer values
724 /// Move unaligned packed integer values
725 /// Move quadword from XMM to MMX technology register
726 /// Move quadword from MMX technology to XMM register
727 mov,
728 /// Move with sign extension
729 movsx,
730 /// Move with zero extension
731 movzx,
732 /// Multiply
733 /// Signed multiplication
734 /// Multiply packed single-precision floating-point values
735 /// Multiply scalar single-precision floating-point values
736 /// Multiply packed double-precision floating-point values
737 /// Multiply scalar double-precision floating-point values
738 /// Multiply packed unsigned doubleword integers
739 /// Multiply packed doubleword integers
740 /// Carry-less multiplication quadword
741 mul,
742 /// Two's complement negation
743 neg,
744 /// No-op
745 /// No operation
746 nop,
747 /// One's complement negation
748 not,
749 /// Logical or
750 /// Bitwise logical or of packed single-precision floating-point values
751 /// Bitwise logical or of packed double-precision floating-point values
752 @"or",
753 /// Output to port
754 /// Output string to port
755 out,
756 /// Spin loop hint
757 /// Timed pause
758 pause,
759 /// Pop
760 pop,
761 /// Return the count of number of bits set to 1
762 popcnt,
763 /// Pop stack into EFLAGS register
764 popf,
765 /// Push
766 push,
767 /// Push EFLAGS register onto the stack
768 pushf,
769 /// Rotate left through carry
770 /// Rotate right through carry
771 rc,
772 /// Read FS segment base
773 /// Read GS segment base
774 /// Read from model specific register
775 /// Read processor ID
776 /// Read protection key rights for user pages
777 /// Read performance-monitoring counters
778 /// Read random number
779 /// Read random seed
780 /// Read shadow stack pointer
781 /// Read time-stamp counter
782 /// Read time-stamp counter and processor ID
783 rd,
784 /// Return
785 /// Return from fast system call
786 /// Interrupt return
787 /// User-interrupt return
788 ret,
789 /// Rotate left
790 /// Rotate right
791 /// Rotate right logical without affecting flags
792 ro,
793 /// Resume from system management mode
794 rsm,
795 /// Arithmetic shift left
796 /// Arithmetic shift right
797 /// Shift left arithmetic without affecting flags
798 sa,
799 /// Store AH into flags
800 sahf,
801 /// Integer subtraction with borrow
802 sbb,
803 /// Scan string
804 sca,
805 /// Send user interprocessor interrupt
806 senduipi,
807 /// Serialize instruction execution
808 serialize,
809 /// Set byte on condition
810 set,
811 /// Logical shift left
812 /// Double precision shift left
813 /// Logical shift right
814 /// Double precision shift right
815 /// Shift left logical without affecting flags
816 /// Shift right logical without affecting flags
817 sh,
818 /// Store interrupt descriptor table register
819 sidt,
820 /// Store local descriptor table register
821 sldt,
822 /// Store machine status word
823 smsw,
824 /// Subtract
825 /// Subtract packed integers
826 /// Subtract packed single-precision floating-point values
827 /// Subtract scalar single-precision floating-point values
828 /// Subtract packed double-precision floating-point values
829 /// Subtract scalar double-precision floating-point values
830 /// Packed single-precision floating-point horizontal subtract
831 /// Packed double-precision floating-point horizontal subtract
832 /// Packed horizontal subtract
833 /// Packed horizontal subtract and saturate
834 sub,
835 /// Set carry flag
836 /// Set direction flag
837 /// Set interrupt flag
838 /// Store binary coded decimal integer and pop
839 /// Store floating-point value
840 /// Store integer
841 /// Store x87 FPU control word
842 /// Store x87 FPU environment
843 /// Store x87 FPU status word
844 /// Store MXCSR register state
845 st,
846 /// Store string
847 sto,
848 /// Test condition
849 /// Logical compare
850 /// Packed bit test
851 @"test",
852 /// Undefined instruction
853 ud,
854 /// User level set up monitor address
855 umonitor,
856 /// Verify a segment for reading
857 /// Verify a segment for writing
858 ver,
859 /// Write to model specific register
860 /// Write to model specific register
861 /// Write to model specific register
862 /// Write to shadow stack
863 /// Write to user shadow stack
864 wr,
865 /// Exchange and add
866 xadd,
867 /// Exchange register/memory with register
868 /// Exchange register contents
869 xch,
870 /// Get value of extended control register
871 xgetbv,
872 /// Table look-up translation
873 xlat,
874 /// Logical exclusive-or
875 /// Bitwise logical xor of packed single-precision floating-point values
876 /// Bitwise logical xor of packed double-precision floating-point values
877 xor,
878
879 // X87
880 /// Compute 2^x-1
881 @"2xm1",
882 /// Absolute value
883 abs,
884 /// Change sign
885 chs,
886 /// Clear exceptions
887 clex,
888 /// Compare floating-point values
889 com,
890 /// Compare floating-point values and set EFLAGS
891 /// Compare scalar ordered single-precision floating-point values
892 /// Compare scalar ordered double-precision floating-point values
893 comi,
894 /// Cosine
895 cos,
896 /// Reverse divide
897 divr,
898 /// Free floating-point register
899 free,
900 /// Initialize floating-point unit
901 init,
902 /// Load binary coded decimal integer
903 /// Load floating-point value
904 /// Load integer
905 /// Load constant
906 /// Load x87 FPU control word
907 /// Load x87 FPU environment
908 /// Load MXCSR register state
909 ld,
910 /// Partial arctangent
911 patan,
912 /// Partial remainder
913 prem,
914 /// Partial tangent
915 ptan,
916 /// Round to integer
917 rndint,
918 /// Restore x87 FPU state
919 /// Restore x87 FPU, MMX, XMM, and MXCSR state
920 rstor,
921 /// Store x87 FPU state
922 /// Save x87 FPU, MMX technology, and MXCSR state
923 save,
924 /// Scale
925 scale,
926 /// Sine
927 sin,
928 /// Sine and cosine
929 sincos,
930 /// Square root
931 /// Square root of packed single-precision floating-point values
932 /// Square root of scalar single-precision floating-point value
933 /// Square root of packed double-precision floating-point values
934 /// Square root of scalar double-precision floating-point value
935 sqrt,
936 /// Store integer with truncation
937 stt,
938 /// Reverse subtract
939 subr,
940 /// Test
941 tst,
942 /// Unordered compare floating-point values
943 ucom,
944 /// Unordered compare floating-point values and set EFLAGS
945 /// Unordered compare scalar single-precision floating-point values
946 /// Unordered compare scalar double-precision floating-point values
947 ucomi,
948 /// Wait
949 /// User level monitor wait
950 wait,
951 /// Examine floating-point
952 xam,
953 /// Extract exponent and significand
954 xtract,
955 /// Compute y * log2x
956 /// Compute y * log2(x + 1)
957 yl2x,
958
959 // MMX
960 /// Pack with signed saturation
961 ackssw,
962 /// Pack with signed saturation
963 ackssd,
964 /// Pack with unsigned saturation
965 ackusw,
966 /// Add packed signed integers with signed saturation
967 adds,
968 /// Add packed unsigned integers with unsigned saturation
969 addus,
970 /// Logical and not
971 /// Bitwise logical and not of packed single-precision floating-point values
972 /// Bitwise logical and not of packed double-precision floating-point values
973 andn,
974 /// Compare packed data for equal
975 cmpeq,
976 /// Compare packed data for greater than
977 cmpgt,
978 /// Empty MMX technology state
979 emms,
980 /// Multiply and add packed signed and unsigned bytes
981 maddubs,
982 /// Multiply and add packed integers
983 maddw,
984 /// Multiply packed signed integers and store low result
985 mull,
986 /// Multiply packed signed integers and store high result
987 mulh,
988 /// Shift packed data left logical
989 sll,
990 /// Shift packed data right arithmetic
991 sra,
992 /// Shift packed data right logical
993 srl,
994 /// Subtract packed signed integers with signed saturation
995 subs,
996 /// Subtract packed unsigned integers with unsigned saturation
997 subus,
998 /// Unpack high data
999 unpckhbw,
1000 /// Unpack high data
1001 unpckhdq,
1002 /// Unpack high data
1003 unpckhwd,
1004 /// Unpack low data
1005 unpcklbw,
1006 /// Unpack low data
1007 unpckldq,
1008 /// Unpack low data
1009 unpcklwd,
1010
1011 // SSE
1012 /// Average packed integers
1013 avg,
1014 /// Convert packed doubleword integers to packed single-precision floating-point values
1015 /// Convert packed doubleword integers to packed double-precision floating-point values
1016 cvtpi2,
1017 /// Convert packed single-precision floating-point values to packed doubleword integers
1018 cvtps2pi,
1019 /// Convert doubleword integer to scalar single-precision floating-point value
1020 /// Convert doubleword integer to scalar double-precision floating-point value
1021 cvtsi2,
1022 /// Convert scalar single-precision floating-point value to doubleword integer
1023 cvtss2si,
1024 /// Convert with truncation packed single-precision floating-point values to packed doubleword integers
1025 cvttps2pi,
1026 /// Convert with truncation scalar single-precision floating-point value to doubleword integer
1027 cvttss2si,
1028 /// Extract byte
1029 /// Extract word
1030 /// Extract doubleword
1031 /// Extract quadword
1032 extr,
1033 /// Insert byte
1034 /// Insert word
1035 /// Insert doubleword
1036 /// Insert quadword
1037 insr,
1038 /// Maximum of packed single-precision floating-point values
1039 /// Maximum of scalar single-precision floating-point values
1040 /// Maximum of packed double-precision floating-point values
1041 /// Maximum of scalar double-precision floating-point values
1042 max,
1043 /// Maximum of packed signed integers
1044 maxs,
1045 /// Maximum of packed unsigned integers
1046 maxu,
1047 /// Minimum of packed single-precision floating-point values
1048 /// Minimum of scalar single-precision floating-point values
1049 /// Minimum of packed double-precision floating-point values
1050 /// Minimum of scalar double-precision floating-point values
1051 min,
1052 /// Minimum of packed signed integers
1053 mins,
1054 /// Minimum of packed unsigned integers
1055 minu,
1056 /// Move aligned packed single-precision floating-point values
1057 /// Move aligned packed double-precision floating-point values
1058 mova,
1059 /// Move high packed single-precision floating-point values
1060 /// Move high packed double-precision floating-point values
1061 movh,
1062 /// Move packed single-precision floating-point values high to low
1063 movhl,
1064 /// Move low packed single-precision floating-point values
1065 /// Move low packed double-precision floating-point values
1066 movl,
1067 /// Move packed single-precision floating-point values low to high
1068 movlh,
1069 /// Move byte mask
1070 /// Extract packed single precision floating-point sign mask
1071 /// Extract packed double precision floating-point sign mask
1072 movmsk,
1073 /// Move unaligned packed single-precision floating-point values
1074 /// Move unaligned packed double-precision floating-point values
1075 movu,
1076 /// Multiply packed unsigned integers and store high result
1077 mulhu,
1078 /// Prefetch data into caches
1079 /// Prefetch data into caches with intent to write
1080 prefetch,
1081 /// Compute sum of absolute differences
1082 sadb,
1083 /// Packed interleave shuffle of quadruplets of single-precision floating-point values
1084 /// Packed interleave shuffle of pairs of double-precision floating-point values
1085 /// Shuffle packed doublewords
1086 /// Shuffle packed words
1087 shuf,
1088 /// Unpack and interleave high packed single-precision floating-point values
1089 /// Unpack and interleave high packed double-precision floating-point values
1090 unpckh,
1091 /// Unpack and interleave low packed single-precision floating-point values
1092 /// Unpack and interleave low packed double-precision floating-point values
1093 unpckl,
1094
1095 // SSE2
1096 /// Convert packed doubleword integers to packed single-precision floating-point values
1097 /// Convert packed doubleword integers to packed double-precision floating-point values
1098 cvtdq2,
1099 /// Convert packed double-precision floating-point values to packed doubleword integers
1100 cvtpd2dq,
1101 /// Convert packed double-precision floating-point values to packed doubleword integers
1102 cvtpd2pi,
1103 /// Convert packed double-precision floating-point values to packed single-precision floating-point values
1104 cvtpd2,
1105 /// Convert packed single-precision floating-point values to packed doubleword integers
1106 cvtps2dq,
1107 /// Convert packed single-precision floating-point values to packed double-precision floating-point values
1108 cvtps2,
1109 /// Convert scalar double-precision floating-point value to doubleword integer
1110 cvtsd2si,
1111 /// Convert scalar double-precision floating-point value to scalar single-precision floating-point value
1112 cvtsd2,
1113 /// Convert scalar single-precision floating-point value to scalar double-precision floating-point value
1114 cvtss2,
1115 /// Convert with truncation packed double-precision floating-point values to packed doubleword integers
1116 cvttpd2dq,
1117 /// Convert with truncation packed double-precision floating-point values to packed doubleword integers
1118 cvttpd2pi,
1119 /// Convert with truncation packed single-precision floating-point values to packed doubleword integers
1120 cvttps2dq,
1121 /// Convert with truncation scalar double-precision floating-point value to doubleword integer
1122 cvttsd2si,
1123 /// Galois field affine transformation inverse
1124 gf2p8affineinvq,
1125 /// Galois field affine transformation
1126 gf2p8affineq,
1127 /// Galois field multiply bytes
1128 gf2p8mul,
1129 /// Shuffle packed high words
1130 shufh,
1131 /// Shuffle packed low words
1132 shufl,
1133 /// Unpack high data
1134 unpckhqdq,
1135 /// Unpack low data
1136 unpcklqdq,
1137
1138 // SSE3
1139 /// Packed single-precision floating-point add/subtract
1140 /// Packed double-precision floating-point add/subtract
1141 addsub,
1142 /// Replicate double floating-point values
1143 movddup,
1144 /// Replicate single floating-point values
1145 movshdup,
1146 /// Replicate single floating-point values
1147 movsldup,
1148
1149 // SSSE3
1150 /// Packed align right
1151 alignr,
1152 /// Packed multiply high with round and scale
1153 mulhrs,
1154 /// Packed sign
1155 sign,
1156
1157 // SSE4.1
1158 /// Pack with unsigned saturation
1159 ackusd,
1160 /// Blend packed single-precision floating-point values
1161 /// Blend scalar single-precision floating-point values
1162 /// Blend packed double-precision floating-point values
1163 /// Blend scalar double-precision floating-point values
1164 /// Blend packed dwords
1165 blend,
1166 /// Variable blend packed single-precision floating-point values
1167 /// Variable blend scalar single-precision floating-point values
1168 /// Variable blend packed double-precision floating-point values
1169 /// Variable blend scalar double-precision floating-point values
1170 blendv,
1171 /// Dot product of packed single-precision floating-point values
1172 /// Dot product of packed double-precision floating-point values
1173 dp,
1174 /// Extract packed floating-point values
1175 /// Extract packed integer values
1176 extract,
1177 /// Insert scalar single-precision floating-point value
1178 /// Insert packed floating-point values
1179 insert,
1180 /// Packed horizontal word minimum
1181 minposu,
1182 /// Packed move with sign extend
1183 movsxb,
1184 movsxd,
1185 movsxw,
1186 /// Packed move with zero extend
1187 movzxb,
1188 movzxd,
1189 movzxw,
1190 /// Round packed single-precision floating-point values
1191 /// Round scalar single-precision floating-point value
1192 /// Round packed double-precision floating-point values
1193 /// Round scalar double-precision floating-point value
1194 round,
1195
1196 // SSE4.2
1197 /// Accumulate CRC32 value
1198 crc32,
1199
1200 // AES
1201 /// Perform one round of an AES decryption flow
1202 /// Perform ten rounds of AES decryption flow with key locker using 128-bit key
1203 /// Perform ten rounds of AES decryption flow with key locker using 256-bit key
1204 /// Perform ten rounds of AES decryption flow with key locker on 8 blocks using 128-bit key
1205 /// Perform ten rounds of AES decryption flow with key locker on 8 blocks using 256-bit key
1206 aesdec,
1207 /// Perform last round of an AES decryption flow
1208 aesdeclast,
1209 /// Perform one round of an AES encryption flow
1210 /// Perform ten rounds of AES encryption flow with key locker using 128-bit key
1211 /// Perform ten rounds of AES encryption flow with key locker using 256-bit key
1212 /// Perform ten rounds of AES encryption flow with key locker on 8 blocks using 128-bit key
1213 /// Perform ten rounds of AES encryption flow with key locker on 8 blocks using 256-bit key
1214 aesenc,
1215 /// Perform last round of an AES encryption flow
1216 aesenclast,
1217 /// Perform the AES InvMixColumn transformation
1218 aesimc,
1219 /// AES round key generation assist
1220 aeskeygenassist,
1221
1222 // SHA
1223 /// Perform four rounds of SHA1 operation
1224 sha1rnds,
1225 /// Calculate SHA1 state variable E after four rounds
1226 sha1nexte,
1227 /// Perform an intermediate calculation for the next four SHA1 message dwords
1228 /// Perform a final calculation for the next four SHA1 message dwords
1229 sha1msg,
1230 /// Perform an intermediate calculation for the next four SHA256 message dwords
1231 /// Perform a final calculation for the next four SHA256 message dwords
1232 sha256msg,
1233 /// Perform two rounds of SHA256 operation
1234 sha256rnds,
1235
1236 // AVX
1237 /// Load with broadcast floating-point data
1238 /// Load integer and broadcast
1239 broadcast,
1240 /// Conditional SIMD packed loads and stores
1241 /// Condition SIMD integer packed loads and stores
1242 maskmov,
1243 /// Permute floating-point values
1244 /// Permute integer values
1245 perm2,
1246 /// Permute in-lane pairs of double-precision floating-point values
1247 /// Permute in-lane quadruples of single-precision floating-point values
1248 permil,
1249
1250 // BMI
1251 /// Bit field extract
1252 bextr,
1253 /// Extract lowest set isolated bit
1254 /// Get mask up to lowest set bit
1255 /// Reset lowest set bit
1256 bls,
1257 /// Count the number of trailing zero bits
1258 tzcnt,
1259
1260 // BMI2
1261 /// Zero high bits starting with specified bit position
1262 bzhi,
1263 /// Parallel bits deposit
1264 pdep,
1265 /// Parallel bits extract
1266 pext,
1267
1268 // F16C
1269 /// Convert 16-bit floating-point values to single-precision floating-point values
1270 cvtph2,
1271 /// Convert single-precision floating-point values to 16-bit floating-point values
1272 cvtps2ph,
1273
1274 // FMA
1275 /// Fused multiply-add of packed single-precision floating-point values
1276 /// Fused multiply-add of scalar single-precision floating-point values
1277 /// Fused multiply-add of packed double-precision floating-point values
1278 /// Fused multiply-add of scalar double-precision floating-point values
1279 fmadd132,
1280 /// Fused multiply-add of packed single-precision floating-point values
1281 /// Fused multiply-add of scalar single-precision floating-point values
1282 /// Fused multiply-add of packed double-precision floating-point values
1283 /// Fused multiply-add of scalar double-precision floating-point values
1284 fmadd213,
1285 /// Fused multiply-add of packed single-precision floating-point values
1286 /// Fused multiply-add of scalar single-precision floating-point values
1287 /// Fused multiply-add of packed double-precision floating-point values
1288 /// Fused multiply-add of scalar double-precision floating-point values
1289 fmadd231,
1290
1291 // AVX2
1292 /// Permute packed doubleword elements
1293 /// Permute packed qword elements
1294 /// Permute double-precision floating-point elements
1295 /// Permute single-precision floating-point elements
1296 perm,
1297 /// Variable bit shift left logical
1298 sllv,
1299 /// Variable bit shift right arithmetic
1300 srav,
1301 /// Variable bit shift right logical
1302 srlv,
1303
1304 // ADX
1305 /// Unsigned integer addition of two operands with overflow flag
1306 ado,
1307
1308 // AESKLE
1309 /// Encode 128-bit key with key locker
1310 /// Encode 256-bit key with key locker
1311 encodekey,
1312 /// Load internal wrapping key with key locker
1313 loadiwkey,
1314
1315 /// A pseudo instruction that requires special lowering.
1316 /// This should be the only tag in this enum that doesn't
1317 /// directly correspond to one or more instruction mnemonics.
1318 pseudo,
1319 };
1320
1321 pub const FixedTag = struct { Fixes, Tag };
1322
1323 pub const Ops = enum(u8) {
1324 /// No data associated with this instruction (only mnemonic is used).
1325 none,
1326 /// Single register operand.
1327 /// Uses `r` payload.
1328 r,
1329 /// Register, register operands.
1330 /// Uses `rr` payload.
1331 rr,
1332 /// Register, register, register operands.
1333 /// Uses `rrr` payload.
1334 rrr,
1335 /// Register, register, register, register operands.
1336 /// Uses `rrrr` payload.
1337 rrrr,
1338 /// Register, register, register, immediate (byte) operands.
1339 /// Uses `rrri` payload.
1340 rrri,
1341 /// Register, register, immediate (sign-extended) operands.
1342 /// Uses `rri` payload.
1343 rri_s,
1344 /// Register, register, immediate (unsigned) operands.
1345 /// Uses `rri` payload.
1346 rri_u,
1347 /// Register, immediate (sign-extended) operands.
1348 /// Uses `ri` payload.
1349 ri_s,
1350 /// Register, immediate (unsigned) operands.
1351 /// Uses `ri` payload.
1352 ri_u,
1353 /// Register, 64-bit unsigned immediate operands.
1354 /// Uses `ri` payload with `i` index of extra data of type `Imm64`.
1355 ri_64,
1356 /// Immediate (sign-extended) operand.
1357 /// Uses `i` payload.
1358 i_s,
1359 /// Immediate (unsigned) operand.
1360 /// Uses `i` payload.
1361 i_u,
1362 /// Immediate (word), immediate (byte) operands.
1363 /// Uses `ii` payload.
1364 ii,
1365 /// Immediate (byte), register operands.
1366 /// Uses `ri` payload.
1367 ir,
1368 /// Register, memory operands.
1369 /// Uses `rx` payload with extra data of type `Memory`.
1370 rm,
1371 /// Register, memory, register operands.
1372 /// Uses `rrx` payload with extra data of type `Memory`.
1373 rmr,
1374 /// Register, memory, immediate (signed word) operands.
1375 /// Uses `rix` payload with extra data of type `Memory`.
1376 rmi,
1377 /// Register, memory, immediate (signed) operands.
1378 /// Uses `rx` payload with extra data of type `Imm32` followed by `Memory`.
1379 rmi_s,
1380 /// Register, memory, immediate (unsigned) operands.
1381 /// Uses `rx` payload with extra data of type `Imm32` followed by `Memory`.
1382 rmi_u,
1383 /// Register, register, memory.
1384 /// Uses `rrix` payload with extra data of type `Memory`.
1385 rrm,
1386 /// Register, register, memory, register.
1387 /// Uses `rrrx` payload with extra data of type `Memory`.
1388 rrmr,
1389 /// Register, register, memory, immediate (byte) operands.
1390 /// Uses `rrix` payload with extra data of type `Memory`.
1391 rrmi,
1392 /// Single memory operand.
1393 /// Uses `x` payload with extra data of type `Memory`.
1394 m,
1395 /// Memory, immediate (sign-extend) operands.
1396 /// Uses `x` payload with extra data of type `Imm32` followed by `Memory`.
1397 mi_s,
1398 /// Memory, immediate (unsigned) operands.
1399 /// Uses `x` payload with extra data of type `Imm32` followed by `Memory`.
1400 mi_u,
1401 /// Memory, register operands.
1402 /// Uses `rx` payload with extra data of type `Memory`.
1403 mr,
1404 /// Memory, register, register operands.
1405 /// Uses `rrx` payload with extra data of type `Memory`.
1406 mrr,
1407 /// Memory, register, immediate (signed word) operands.
1408 /// Uses `rix` payload with extra data of type `Memory`.
1409 mri,
1410 /// References another Mir instruction directly.
1411 /// Uses `inst` payload.
1412 inst,
1413 /// References a nav.
1414 /// Uses `nav` payload.
1415 nav,
1416 /// References an uav.
1417 /// Uses `uav` payload.
1418 uav,
1419 /// References a lazy symbol.
1420 /// Uses `lazy_sym` payload.
1421 lazy_sym,
1422 /// References an external symbol.
1423 /// Uses `extern_func` payload.
1424 extern_func,
1425
1426 // Pseudo instructions:
1427
1428 /// Conditional move if zero flag set and parity flag not set
1429 /// Clobbers the source operand!
1430 /// Uses `rr` payload.
1431 pseudo_cmov_z_and_np_rr,
1432 /// Conditional move if zero flag not set or parity flag set
1433 /// Uses `rr` payload.
1434 pseudo_cmov_nz_or_p_rr,
1435 /// Conditional move if zero flag not set or parity flag set
1436 /// Uses `rx` payload.
1437 pseudo_cmov_nz_or_p_rm,
1438 /// Set byte if zero flag set and parity flag not set
1439 /// Requires a scratch register!
1440 /// Uses `rr` payload.
1441 pseudo_set_z_and_np_r,
1442 /// Set byte if zero flag set and parity flag not set
1443 /// Requires a scratch register!
1444 /// Uses `rx` payload.
1445 pseudo_set_z_and_np_m,
1446 /// Set byte if zero flag not set or parity flag set
1447 /// Requires a scratch register!
1448 /// Uses `rr` payload.
1449 pseudo_set_nz_or_p_r,
1450 /// Set byte if zero flag not set or parity flag set
1451 /// Requires a scratch register!
1452 /// Uses `rx` payload.
1453 pseudo_set_nz_or_p_m,
1454 /// Jump if zero flag set and parity flag not set
1455 /// Uses `inst` payload.
1456 pseudo_j_z_and_np_inst,
1457 /// Jump if zero flag not set or parity flag set
1458 /// Uses `inst` payload.
1459 pseudo_j_nz_or_p_inst,
1460
1461 /// Probe alignment
1462 /// Uses `ri` payload.
1463 pseudo_probe_align_ri_s,
1464 /// Probe adjust unrolled
1465 /// Uses `ri` payload.
1466 pseudo_probe_adjust_unrolled_ri_s,
1467 /// Probe adjust setup
1468 /// Uses `rri` payload.
1469 pseudo_probe_adjust_setup_rri_s,
1470 /// Probe adjust loop
1471 /// Uses `rr` payload.
1472 pseudo_probe_adjust_loop_rr,
1473
1474 /// Push registers
1475 /// Uses `reg_list` payload.
1476 pseudo_push_reg_list,
1477 /// Pop registers
1478 /// Uses `reg_list` payload.
1479 pseudo_pop_reg_list,
1480
1481 /// Define cfa rule as offset from register.
1482 /// Uses `ri` payload.
1483 pseudo_cfi_def_cfa_ri_s,
1484 /// Modify cfa rule register.
1485 /// Uses `r` payload.
1486 pseudo_cfi_def_cfa_register_r,
1487 /// Modify cfa rule offset.
1488 /// Uses `i` payload.
1489 pseudo_cfi_def_cfa_offset_i_s,
1490 /// Offset cfa rule offset.
1491 /// Uses `i` payload.
1492 pseudo_cfi_adjust_cfa_offset_i_s,
1493 /// Define register rule as stored at offset from cfa.
1494 /// Uses `ri` payload.
1495 pseudo_cfi_offset_ri_s,
1496 /// Define register rule as offset from cfa.
1497 /// Uses `ri` payload.
1498 pseudo_cfi_val_offset_ri_s,
1499 /// Define register rule as stored at offset from cfa rule register.
1500 /// Uses `ri` payload.
1501 pseudo_cfi_rel_offset_ri_s,
1502 /// Define register rule as register.
1503 /// Uses `rr` payload.
1504 pseudo_cfi_register_rr,
1505 /// Define register rule from initial.
1506 /// Uses `r` payload.
1507 pseudo_cfi_restore_r,
1508 /// Define register rule as undefined.
1509 /// Uses `r` payload.
1510 pseudo_cfi_undefined_r,
1511 /// Define register rule as itself.
1512 /// Uses `r` payload.
1513 pseudo_cfi_same_value_r,
1514 /// Push cfi state.
1515 pseudo_cfi_remember_state_none,
1516 /// Pop cfi state.
1517 pseudo_cfi_restore_state_none,
1518 /// Raw cfi bytes.
1519 /// Uses `bytes` payload.
1520 pseudo_cfi_escape_bytes,
1521
1522 /// End of prologue.
1523 /// Uses `none` payload.
1524 pseudo_dbg_prologue_end_none,
1525 /// Update debug line with is_stmt register set.
1526 /// Uses `line_column` payload.
1527 pseudo_dbg_line_stmt_line_column,
1528 /// Update debug line with is_stmt register clear.
1529 /// Uses `line_column` payload.
1530 pseudo_dbg_line_line_column,
1531 /// Start of epilogue.
1532 /// Uses `line_column` payload.
1533 pseudo_dbg_epilogue_begin_line_column,
1534 /// Start of lexical block.
1535 /// Uses `none` payload.
1536 pseudo_dbg_enter_block_none,
1537 /// End of lexical block.
1538 /// Uses `none` payload.
1539 pseudo_dbg_leave_block_none,
1540 /// Start of inline function.
1541 /// Uses `ip_index` payload.
1542 pseudo_dbg_enter_inline_func,
1543 /// End of inline function.
1544 /// Uses `ip_index` payload.
1545 pseudo_dbg_leave_inline_func,
1546 /// End of function.
1547 /// Uses `none` payload.
1548 pseudo_dbg_end_none,
1549 /// Local argument.
1550 /// Uses `none` payload.
1551 pseudo_dbg_arg_none,
1552 /// Local argument.
1553 /// Uses `i` payload.
1554 pseudo_dbg_arg_i_s,
1555 /// Local argument.
1556 /// Uses `i` payload.
1557 pseudo_dbg_arg_i_u,
1558 /// Local argument.
1559 /// Uses `i64` payload.
1560 pseudo_dbg_arg_i_64,
1561 /// Local argument.
1562 /// Uses `ro` payload.
1563 pseudo_dbg_arg_ro,
1564 /// Local argument.
1565 /// Uses `fa` payload.
1566 pseudo_dbg_arg_fa,
1567 /// Local argument.
1568 /// Uses `x` payload with extra data of type `Memory`.
1569 pseudo_dbg_arg_m,
1570 /// Local argument.
1571 /// Uses `ip_index` payload.
1572 pseudo_dbg_arg_val,
1573 /// Remaining arguments are varargs.
1574 pseudo_dbg_var_args_none,
1575 /// Local variable.
1576 /// Uses `none` payload.
1577 pseudo_dbg_var_none,
1578 /// Local variable.
1579 /// Uses `i` payload.
1580 pseudo_dbg_var_i_s,
1581 /// Local variable.
1582 /// Uses `i` payload.
1583 pseudo_dbg_var_i_u,
1584 /// Local variable.
1585 /// Uses `i64` payload.
1586 pseudo_dbg_var_i_64,
1587 /// Local variable.
1588 /// Uses `ro` payload.
1589 pseudo_dbg_var_ro,
1590 /// Local variable.
1591 /// Uses `fa` payload.
1592 pseudo_dbg_var_fa,
1593 /// Local variable.
1594 /// Uses `x` payload with extra data of type `Memory`.
1595 pseudo_dbg_var_m,
1596 /// Local variable.
1597 /// Uses `ip_index` payload.
1598 pseudo_dbg_var_val,
1599
1600 /// Tombstone
1601 /// Emitter should skip this instruction.
1602 pseudo_dead_none,
1603 };
1604
1605 pub const Data = union {
1606 none: struct {
1607 fixes: Fixes = ._,
1608 },
1609 /// References another Mir instruction.
1610 inst: struct {
1611 fixes: Fixes = ._,
1612 inst: Index,
1613 },
1614 /// A 32-bit immediate value.
1615 i64: u64,
1616 i: struct {
1617 fixes: Fixes = ._,
1618 i: u32,
1619 },
1620 ii: struct {
1621 fixes: Fixes = ._,
1622 i1: u16,
1623 i2: u8,
1624 },
1625 r: struct {
1626 fixes: Fixes = ._,
1627 r1: Register,
1628 },
1629 rr: struct {
1630 fixes: Fixes = ._,
1631 r1: Register,
1632 r2: Register,
1633 },
1634 rrr: struct {
1635 fixes: Fixes = ._,
1636 r1: Register,
1637 r2: Register,
1638 r3: Register,
1639 },
1640 rrrr: struct {
1641 fixes: Fixes = ._,
1642 r1: Register,
1643 r2: Register,
1644 r3: Register,
1645 r4: Register,
1646 },
1647 rrri: struct {
1648 fixes: Fixes = ._,
1649 r1: Register,
1650 r2: Register,
1651 r3: Register,
1652 i: u8,
1653 },
1654 rri: struct {
1655 fixes: Fixes = ._,
1656 r1: Register,
1657 r2: Register,
1658 i: u32,
1659 },
1660 /// Register, immediate.
1661 ri: struct {
1662 fixes: Fixes = ._,
1663 r1: Register,
1664 i: u32,
1665 },
1666 /// Register, followed by custom payload found in extra.
1667 rx: struct {
1668 fixes: Fixes = ._,
1669 r1: Register,
1670 payload: u32,
1671 },
1672 /// Register, register, followed by Custom payload found in extra.
1673 rrx: struct {
1674 fixes: Fixes = ._,
1675 r1: Register,
1676 r2: Register,
1677 payload: u32,
1678 },
1679 /// Register, register, register, followed by Custom payload found in extra.
1680 rrrx: struct {
1681 fixes: Fixes = ._,
1682 r1: Register,
1683 r2: Register,
1684 r3: Register,
1685 payload: u32,
1686 },
1687 /// Register, byte immediate, followed by Custom payload found in extra.
1688 rix: struct {
1689 fixes: Fixes = ._,
1690 r1: Register,
1691 i: i16,
1692 payload: u32,
1693 },
1694 /// Register, register, byte immediate, followed by Custom payload found in extra.
1695 rrix: struct {
1696 fixes: Fixes = ._,
1697 r1: Register,
1698 r2: Register,
1699 i: u8,
1700 payload: u32,
1701 },
1702 /// Custom payload found in extra.
1703 x: struct {
1704 fixes: Fixes = ._,
1705 payload: u32,
1706 },
1707 bytes: struct {
1708 payload: u32,
1709 len: u32,
1710
1711 pub fn get(bytes: @This(), mir: Mir) []const u8 {
1712 return std.mem.sliceAsBytes(mir.extra[bytes.payload..])[0..bytes.len];
1713 }
1714 },
1715 fa: bits.FrameAddr,
1716 ro: bits.RegisterOffset,
1717 nav: bits.NavOffset,
1718 uav: InternPool.Key.Ptr.BaseAddr.Uav,
1719 lazy_sym: link.File.LazySymbol,
1720 extern_func: Mir.NullTerminatedString,
1721 /// Debug line and column position
1722 line_column: struct {
1723 line: u32,
1724 column: u32,
1725 },
1726 ip_index: InternPool.Index,
1727 /// Register list
1728 reg_list: RegisterList,
1729 };
1730
1731 comptime {
1732 if (!std.debug.runtime_safety) {
1733 // Make sure we don't accidentally make instructions bigger than expected.
1734 // Note that in safety builds, Zig is allowed to insert a secret field for safety checks.
1735 assert(@sizeOf(Data) == 8);
1736 }
1737 const Mnemonic = @import("Encoding.zig").Mnemonic;
1738 if (@typeInfo(Mnemonic).@"enum".field_names.len != 978 or
1739 @typeInfo(Fixes).@"enum".field_names.len != 231 or
1740 @typeInfo(Tag).@"enum".field_names.len != 251)
1741 {
1742 const cond_src = (struct {
1743 fn src() std.lang.SourceLocation {
1744 return @src();
1745 }
1746 }).src();
1747 @setEvalBranchQuota(2_000_000);
1748 for (@typeInfo(Mnemonic).@"enum".field_names) |mnemonic_name| {
1749 if (mnemonic_name[0] == '.') continue;
1750 for (@typeInfo(Fixes).@"enum".field_names) |fixes_name| {
1751 const pattern = fixes_name[if (std.mem.findScalar(u8, fixes_name, ' ')) |index| index + " ".len else 0..];
1752 const wildcard_index = std.mem.findScalar(u8, pattern, '_').?;
1753 const mnem_prefix = pattern[0..wildcard_index];
1754 const mnem_suffix = pattern[wildcard_index + "_".len ..];
1755 if (!std.mem.startsWith(u8, mnemonic_name, mnem_prefix)) continue;
1756 if (!std.mem.endsWith(u8, mnemonic_name, mnem_suffix)) continue;
1757 if (@hasField(
1758 Tag,
1759 mnemonic_name[mnem_prefix.len .. mnemonic_name.len - mnem_suffix.len],
1760 )) break;
1761 } else @compileError("'" ++ mnemonic_name ++ "' is not encodable in Mir");
1762 }
1763 @compileError(std.fmt.comptimePrint(
1764 \\All mnemonics are encodable in Mir! You may now change the condition at {s}:{d} to:
1765 \\if (@typeInfo(Mnemonic).@"enum".field_names.len != {d} or
1766 \\ @typeInfo(Fixes).@"enum".field_names.len != {d} or
1767 \\ @typeInfo(Tag).@"enum".field_names.len != {d})
1768 , .{
1769 cond_src.file,
1770 cond_src.line - 6,
1771 @typeInfo(Mnemonic).@"enum".field_names.len,
1772 @typeInfo(Fixes).@"enum".field_names.len,
1773 @typeInfo(Tag).@"enum".field_names.len,
1774 }));
1775 }
1776 }
1777};
1778
1779/// Used in conjunction with payload to transfer a list of used registers in a compact manner.
1780pub const RegisterList = struct {
1781 bitset: BitSet,
1782
1783 const BitSet = std.bit_set.Integer(32);
1784 const Self = @This();
1785
1786 pub const empty: RegisterList = .{ .bitset = .empty };
1787
1788 fn getIndexForReg(registers: []const Register, reg: Register) BitSet.MaskInt {
1789 for (registers, 0..) |cpreg, i| {
1790 if (reg.id() == cpreg.id()) return @intCast(i);
1791 }
1792 unreachable; // register not in input register list!
1793 }
1794
1795 pub fn push(self: *Self, registers: []const Register, reg: Register) void {
1796 const index = getIndexForReg(registers, reg);
1797 self.bitset.set(index);
1798 }
1799
1800 pub fn isSet(self: Self, registers: []const Register, reg: Register) bool {
1801 const index = getIndexForReg(registers, reg);
1802 return self.bitset.isSet(index);
1803 }
1804
1805 pub fn iterator(self: Self, comptime options: std.bit_set.IteratorOptions) BitSet.Iterator(options) {
1806 return self.bitset.iterator(options);
1807 }
1808
1809 pub fn count(self: Self) i32 {
1810 return @intCast(self.bitset.count());
1811 }
1812
1813 pub fn size(self: Self, target: *const std.Target) i32 {
1814 return @intCast(self.bitset.count() * @as(u4, switch (target.cpu.arch) {
1815 else => unreachable,
1816 .x86 => 4,
1817 .x86_64 => 8,
1818 }));
1819 }
1820};
1821
1822pub const NullTerminatedString = enum(u32) {
1823 none = std.math.maxInt(u32),
1824 _,
1825
1826 pub fn toSlice(nts: NullTerminatedString, mir: *const Mir) ?[:0]const u8 {
1827 if (nts == .none) return null;
1828 const string_bytes = mir.string_bytes[@backingInt(nts)..];
1829 return string_bytes[0..std.mem.findScalar(u8, string_bytes, 0).? :0];
1830 }
1831};
1832
1833pub const Local = struct {
1834 name: NullTerminatedString,
1835 type: InternPool.Index,
1836};
1837
1838pub const Imm32 = struct {
1839 imm: u32,
1840};
1841
1842pub const Imm64 = struct {
1843 msb: u32,
1844 lsb: u32,
1845
1846 pub fn encode(v: u64) Imm64 {
1847 return .{
1848 .msb = @truncate(v >> 32),
1849 .lsb = @truncate(v),
1850 };
1851 }
1852
1853 pub fn decode(imm: Imm64) u64 {
1854 var res: u64 = 0;
1855 res |= @as(u64, @intCast(imm.msb)) << 32;
1856 res |= @as(u64, @intCast(imm.lsb));
1857 return res;
1858 }
1859};
1860
1861pub const Memory = struct {
1862 info: Info,
1863 base: u32,
1864 off: u32,
1865 extra: u32,
1866
1867 pub const Info = packed struct(u32) {
1868 base: @typeInfo(bits.Memory.Base).@"union".tag_type.?,
1869 mod: @typeInfo(bits.Memory.Mod).@"union".tag_type.?,
1870 size: bits.Memory.Size,
1871 index: Register,
1872 scale: bits.Memory.Scale,
1873 _: u13 = undefined,
1874 };
1875
1876 pub fn encode(mem: bits.Memory) Memory {
1877 return .{
1878 .info = .{
1879 .base = mem.base,
1880 .mod = mem.mod,
1881 .size = switch (mem.mod) {
1882 .rm => |rm| rm.size,
1883 .off => undefined,
1884 },
1885 .index = switch (mem.mod) {
1886 .rm => |rm| rm.index,
1887 .off => undefined,
1888 },
1889 .scale = switch (mem.mod) {
1890 .rm => |rm| rm.scale,
1891 .off => undefined,
1892 },
1893 },
1894 .base = switch (mem.base) {
1895 .none, .table => undefined,
1896 .reg => |reg| @backingInt(reg),
1897 .frame => |frame_index| @backingInt(frame_index),
1898 .rip_inst => |inst_index| inst_index,
1899 .nav => |nav| @backingInt(nav),
1900 .uav => |uav| @backingInt(uav.val),
1901 .lazy_sym => |lazy_sym| @backingInt(lazy_sym.ty),
1902 .extern_func => |extern_func| @backingInt(extern_func),
1903 },
1904 .off = switch (mem.mod) {
1905 .rm => |rm| @bitCast(rm.disp),
1906 .off => |off| @truncate(off),
1907 },
1908 .extra = switch (mem.mod) {
1909 .rm => switch (mem.base) {
1910 else => undefined,
1911 .uav => |uav| @backingInt(uav.orig_ty),
1912 .lazy_sym => |lazy_sym| @backingInt(lazy_sym.kind),
1913 },
1914 .off => switch (mem.base) {
1915 .reg => @intCast(mem.mod.off >> 32),
1916 else => unreachable,
1917 },
1918 },
1919 };
1920 }
1921
1922 pub fn decode(mem: Memory) encoder.Instruction.Memory {
1923 switch (mem.info.mod) {
1924 .rm => {
1925 if (mem.info.base == .reg and @as(Register, @fromBackingInt(@intCast(mem.base))) == .rip) {
1926 assert(mem.info.index == .none and mem.info.scale == .@"1");
1927 return encoder.Instruction.Memory.initRip(mem.info.size, @bitCast(mem.off));
1928 }
1929 return encoder.Instruction.Memory.initSib(mem.info.size, .{
1930 .disp = @bitCast(mem.off),
1931 .base = switch (mem.info.base) {
1932 .none => .none,
1933 .reg => .{ .reg = @fromBackingInt(@intCast(mem.base)) },
1934 .frame => .{ .frame = @fromBackingInt(@intCast(mem.base)) },
1935 .table => .table,
1936 .rip_inst => .{ .rip_inst = mem.base },
1937 .nav => .{ .nav = @fromBackingInt(@intCast(mem.base)) },
1938 .uav => .{ .uav = .{ .val = @fromBackingInt(@intCast(mem.base)), .orig_ty = @fromBackingInt(@intCast(mem.extra)) } },
1939 .lazy_sym => .{ .lazy_sym = .{ .kind = @fromBackingInt(@intCast(mem.extra)), .ty = @fromBackingInt(@intCast(mem.base)) } },
1940 .extern_func => .{ .extern_func = @fromBackingInt(@intCast(mem.base)) },
1941 },
1942 .scale_index = switch (mem.info.index) {
1943 .none => null,
1944 else => |index| .{ .scale = switch (mem.info.scale) {
1945 inline else => |scale| comptime std.fmt.parseInt(
1946 u4,
1947 @tagName(scale),
1948 10,
1949 ) catch unreachable,
1950 }, .index = index },
1951 },
1952 });
1953 },
1954 .off => {
1955 assert(mem.info.base == .reg);
1956 return encoder.Instruction.Memory.initMoffs(
1957 @fromBackingInt(@intCast(mem.base)),
1958 @as(u64, mem.extra) << 32 | mem.off,
1959 );
1960 },
1961 }
1962 }
1963};
1964
1965pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {
1966 mir.instructions.deinit(gpa);
1967 gpa.free(mir.extra);
1968 gpa.free(mir.string_bytes);
1969 gpa.free(mir.locals);
1970 gpa.free(mir.table);
1971 gpa.free(mir.memoized_encodings);
1972 mir.frame_locs.deinit(gpa);
1973 mir.* = undefined;
1974}
1975
1976pub fn emit(
1977 mir: Mir,
1978 lf: *link.File,
1979 pt: Zcu.PerThread,
1980 func_index: InternPool.Index,
1981 atom_id: link.File.AtomId,
1982 w: *std.Io.Writer,
1983 debug_output: link.File.DebugInfoOutput,
1984) link.EmitError!void {
1985 const zcu = pt.zcu;
1986 const comp = zcu.comp;
1987 const gpa = comp.gpa;
1988 const func = zcu.funcInfo(func_index);
1989 const fn_info = zcu.typeToFunc(.fromInterned(func.ty)).?;
1990 const nav = func.owner_nav;
1991 const mod = zcu.navFileScope(nav).mod.?;
1992 var em: Emit = .{
1993 .lower = .{
1994 .target = &mod.resolved_target.result,
1995 .allocator = gpa,
1996 .mir = mir,
1997 .cc = fn_info.cc,
1998 .src_loc = zcu.navSrcLoc(nav),
1999 },
2000 .bin_file = lf,
2001 .pt = pt,
2002 .pic = mod.pic,
2003 .atom_id = atom_id,
2004 .debug_output = debug_output,
2005 .w = w,
2006
2007 .prev_di_loc = .{
2008 .line = func.lbrace_line,
2009 .column = func.lbrace_column,
2010 .is_stmt = switch (debug_output) {
2011 .dwarf => |dwarf| dwarf.dwarf.debug_line.header.default_is_stmt,
2012 .dwarf2 => |dwarf| dwarf.wip_nav.dwarf.debug_line.header.default_is_stmt,
2013 .eh_frame, .none => undefined,
2014 },
2015 },
2016 .prev_di_pc = 0,
2017
2018 .code_offset_mapping = .empty,
2019 .relocs = .empty,
2020 .table_relocs = .empty,
2021 };
2022 defer em.deinit();
2023 em.emitMir() catch |err| switch (err) {
2024 error.LowerFail, error.EmitFail => return zcu.codegenFailMsg(nav, em.lower.err_msg.?),
2025 error.InvalidInstruction, error.CannotEncode => return zcu.codegenFail(nav, "emit MIR failed: {s} (Zig compiler bug)", .{@errorName(err)}),
2026 else => return zcu.codegenFail(nav, "emit MIR failed: {s}", .{@errorName(err)}),
2027 error.AlreadyReported, error.Canceled, error.WriteFailed => |e| return e,
2028 };
2029}
2030
2031pub fn emitLazy(
2032 mir: Mir,
2033 lf: *link.File,
2034 pt: Zcu.PerThread,
2035 lazy_sym: link.File.LazySymbol,
2036 atom_id: link.File.AtomId,
2037 w: *std.Io.Writer,
2038 debug_output: link.File.DebugInfoOutput,
2039) link.EmitError!void {
2040 const zcu = pt.zcu;
2041 const comp = zcu.comp;
2042 const gpa = comp.gpa;
2043 const mod = comp.root_mod;
2044 var em: Emit = .{
2045 .lower = .{
2046 .target = &mod.resolved_target.result,
2047 .allocator = gpa,
2048 .mir = mir,
2049 .cc = .auto,
2050 .src_loc = Zcu.Type.fromInterned(lazy_sym.ty).srcLocOrNull(zcu) orelse .unneeded,
2051 },
2052 .bin_file = lf,
2053 .pt = pt,
2054 .pic = mod.pic,
2055 .atom_id = atom_id,
2056 .debug_output = debug_output,
2057 .w = w,
2058
2059 .prev_di_loc = undefined,
2060 .prev_di_pc = undefined,
2061
2062 .code_offset_mapping = .empty,
2063 .relocs = .empty,
2064 .table_relocs = .empty,
2065 };
2066 defer em.deinit();
2067 em.emitMir() catch |err| switch (err) {
2068 error.LowerFail, error.EmitFail => return zcu.codegenFailTypeMsg(lazy_sym.ty, em.lower.err_msg.?),
2069 error.InvalidInstruction, error.CannotEncode => return zcu.codegenFailType(lazy_sym.ty, "emit MIR failed: {s} (Zig compiler bug)", .{@errorName(err)}),
2070 else => return zcu.codegenFailType(lazy_sym.ty, "emit MIR failed: {s}", .{@errorName(err)}),
2071 error.AlreadyReported, error.Canceled, error.WriteFailed => |e| return e,
2072 };
2073}
2074
2075pub fn extraData(mir: Mir, comptime T: type, index: u32) struct { data: T, end: u32 } {
2076 const info = @typeInfo(T).@"struct";
2077 var i: u32 = index;
2078 var result: T = undefined;
2079 inline for (info.field_names, info.field_types) |field_name, field_type| {
2080 @field(result, field_name) = switch (field_type) {
2081 u32 => mir.extra[i],
2082 i32, Memory.Info => @bitCast(mir.extra[i]),
2083 bits.FrameIndex => @fromBackingInt(@intCast(mir.extra[i])),
2084 else => @compileError("bad field type: " ++ field_name ++ ": " ++ @typeName(field_type)),
2085 };
2086 i += 1;
2087 }
2088 return .{
2089 .data = result,
2090 .end = i,
2091 };
2092}
2093
2094pub const FrameLoc = struct {
2095 base: Register,
2096 disp: i32,
2097};
2098
2099pub fn resolveFrameAddr(mir: Mir, frame_addr: bits.FrameAddr) bits.RegisterOffset {
2100 const frame_loc = mir.frame_locs.get(@backingInt(frame_addr.index));
2101 return .{ .reg = frame_loc.base, .off = frame_loc.disp + frame_addr.off };
2102}
2103
2104pub fn resolveMemoryExtra(mir: Mir, payload: u32) Memory {
2105 const mem = mir.extraData(Mir.Memory, payload).data;
2106 return switch (mem.info.base) {
2107 .none, .reg, .table, .rip_inst, .nav, .uav, .lazy_sym, .extern_func => mem,
2108 .frame => if (mir.frame_locs.len > 0) .{
2109 .info = .{
2110 .base = .reg,
2111 .mod = mem.info.mod,
2112 .size = mem.info.size,
2113 .index = mem.info.index,
2114 .scale = mem.info.scale,
2115 },
2116 .base = @backingInt(mir.frame_locs.items(.base)[mem.base]),
2117 .off = @bitCast(mir.frame_locs.items(.disp)[mem.base] + @as(i32, @bitCast(mem.off))),
2118 .extra = mem.extra,
2119 } else mem,
2120 };
2121}
2122
2123const assert = std.debug.assert;
2124const bits = @import("bits.zig");
2125const builtin = @import("builtin");
2126const encoder = @import("encoder.zig");
2127const std = @import("std");
2128
2129const InternPool = @import("../../InternPool.zig");
2130const Mir = @This();
2131const Register = bits.Register;
2132const Emit = @import("Emit.zig");
2133const codegen = @import("../../codegen.zig");
2134const link = @import("../../link.zig");
2135const Zcu = @import("../../Zcu.zig");