From 32c6a5967b6d47435578d370c25151fb0b4e0150 Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Wed, 19 Aug 2026 16:40:53 -0700 Subject: [PATCH 1/9] Print undecodable bytes instead of truncating disassembly Capstone's `disasm_all` stops at the first instruction it can't decode and returns success with whatever it managed to decode. We never compared that against the block length, so that instruction and everything after it vanished from the listing. For `precise-output` filetests the truncated output gets blessed, and the test then passes without asserting anything about those bytes. This is reachable today: the bundled capstone can't decode AVX-VNNI, so a function containing `vpdpbusd` was silently dropping five instructions. It is also a precursor to the APX work, whose EVEX map 4 encodings hit the same path; those filetests were being blessed as truncated output and so verified nothing about the instructions they were added to cover. Print the leftover bytes as `.byte`, keeping the reloc and trap annotations. s390x expectations already look like this. Resyncing after the bad instruction isn't possible on x86 without knowing its length, and guessing yields plausible but wrong instructions. The updated expectations only gain lines; nothing already printed changed. Those blocks end in constant pool data that capstone was already rendering as nonsense. prtest:full --- cranelift/codegen/src/machinst/mod.rs | 38 ++++++++++++++ .../filetests/filetests/isa/x64/branches.clif | 4 ++ .../isa/x64/disas-undecodable-bytes.clif | 52 +++++++++++++++++++ .../filetests/isa/x64/f128const.clif | 1 + .../filetests/filetests/isa/x64/fcvt-avx.clif | 8 +++ .../filetests/filetests/isa/x64/fcvt.clif | 9 ++++ .../filetests/isa/x64/float-avx.clif | 2 + .../filetests/filetests/isa/x64/i128.clif | 1 + .../filetests/isa/x64/immediates.clif | 1 + .../filetests/filetests/isa/x64/mul.clif | 1 + .../filetests/isa/x64/narrowing.clif | 2 + .../filetests/isa/x64/select-issue-3744.clif | 1 + .../filetests/isa/x64/shuffle-avx512.clif | 2 + .../filetests/isa/x64/simd-arith-avx.clif | 41 +++++++++++++++ .../filetests/isa/x64/simd-bitselect.clif | 2 + .../isa/x64/simd-bitwise-compile.clif | 27 ++++++++++ .../isa/x64/simd-lane-access-compile.clif | 5 ++ .../filetests/isa/x64/simd-pairwise-add.clif | 1 + .../filetests/isa/x64/sqmul_round_sat.clif | 1 + .../filetests/filetests/isa/x64/uunarrow.clif | 4 ++ 20 files changed, 203 insertions(+) create mode 100644 cranelift/filetests/filetests/isa/x64/disas-undecodable-bytes.clif diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs index 5833047778ef..5b9b16a94aa1 100644 --- a/cranelift/codegen/src/machinst/mod.rs +++ b/cranelift/codegen/src/machinst/mod.rs @@ -703,6 +703,44 @@ impl CompiledCodeBase { writeln!(buf)?; } + + // `disasm_all` stops at the first instruction it cannot decode and + // reports success rather than an error, so without this the rest of + // the block would silently vanish from the listing. That matters + // most for `precise-output` filetests, whose expectations would + // then assert nothing at all about those bytes. Print them as + // `.byte` directives instead, the same form capstone itself + // produces for undecodable s390x instructions. + let decoded: usize = insns.iter().map(|i| i.bytes().len()).sum(); + for (chunk_idx, chunk) in buffer[decoded..].chunks(8).enumerate() { + let addr = start as u64 + decoded as u64 + (chunk_idx * 8) as u64; + let chunk_end = addr + chunk.len() as u64; + let contains = |off| addr <= off && off < chunk_end; + + write!(buf, " .byte ")?; + for (i, byte) in chunk.iter().enumerate() { + if i > 0 { + write!(buf, ", ")?; + } + write!(buf, "{byte:#04x}")?; + } + + for reloc in relocs.iter().filter(|reloc| contains(reloc.offset as u64)) { + write!( + buf, + " ; reloc_external {} {} {}", + reloc.kind, + reloc.target.display(params), + reloc.addend, + )?; + } + + if let Some(trap) = traps.iter().find(|trap| contains(trap.offset as u64)) { + write!(buf, " ; trap: {}", trap.code)?; + } + + writeln!(buf)?; + } } return Ok(buf); diff --git a/cranelift/filetests/filetests/isa/x64/branches.clif b/cranelift/filetests/filetests/isa/x64/branches.clif index bb2191bc31c9..27a30703047b 100644 --- a/cranelift/filetests/filetests/isa/x64/branches.clif +++ b/cranelift/filetests/filetests/isa/x64/branches.clif @@ -1003,6 +1003,9 @@ block5(v5: i32): ; movslq (%rcx, %r10, 4), %rax ; addq %rax, %rcx ; jmpq *%rcx +; .byte 0x2f, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00 +; .byte 0x24, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00 +; .byte 0x3a, 0x00, 0x00, 0x00 ; block2: ; offset 0x38 ; jmp 0x48 ; block3: ; offset 0x3d @@ -1089,6 +1092,7 @@ block1(v5: i32): ; addb %al, (%rax) ; subb $0, %al ; addb %al, (%rax) +; .byte 0x2f, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00 ; block2: ; offset 0x4d ; movq %r11, %rax ; jmp 0x68 diff --git a/cranelift/filetests/filetests/isa/x64/disas-undecodable-bytes.clif b/cranelift/filetests/filetests/isa/x64/disas-undecodable-bytes.clif new file mode 100644 index 000000000000..2f48b8956561 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/disas-undecodable-bytes.clif @@ -0,0 +1,52 @@ +test compile precise-output +set unwind_info=false +target x86_64 has_avx has_avx_vnni + +;; The bundled capstone build cannot decode AVX-VNNI, and capstone stops at the +;; first instruction it does not recognize rather than reporting an error. The +;; remaining bytes are therefore printed as `.byte` directives so that they stay +;; visible here instead of silently disappearing from this expectation. Note +;; that x86 instruction length cannot be determined without decoding, so there +;; is no way to resynchronize: everything after `vpdpbusd`, including the +;; epilogue, is covered by the byte dump. + +function %vpdpbusd_i8x16(i8x16, i8x16, i32x4) -> i32x4 { +block0(v0: i8x16, v1: i8x16, v2: i32x4): + v3 = swiden_low v0 + v4 = uwiden_low v1 + v5 = imul v3, v4 + v6 = swiden_high v0 + v7 = uwiden_high v1 + v8 = imul v6, v7 + v9 = swiden_low v5 + v10 = swiden_high v5 + v11 = iadd_pairwise v9, v10 + v12 = swiden_low v8 + v13 = swiden_high v8 + v14 = iadd_pairwise v12, v13 + v15 = iadd_pairwise v11, v14 + v16 = iadd v15, v2 + return v16 +} + +; VCode: +; pushq %rbp +; movq %rsp, %rbp +; block0: +; movdqa %xmm0, %xmm5 +; movdqa %xmm2, %xmm0 +; vpdpbusd %xmm5, %xmm1, %xmm0 +; movq %rbp, %rsp +; popq %rbp +; retq +; +; Disassembled: +; block0: ; offset 0x0 +; pushq %rbp +; movq %rsp, %rbp +; block1: ; offset 0x4 +; movdqa %xmm0, %xmm5 +; movdqa %xmm2, %xmm0 +; .byte 0xc4, 0xe2, 0x71, 0x50, 0xc5, 0x48, 0x89, 0xec +; .byte 0x5d, 0xc3 + diff --git a/cranelift/filetests/filetests/isa/x64/f128const.clif b/cranelift/filetests/filetests/isa/x64/f128const.clif index aa6500d02b39..26d3bac8e745 100644 --- a/cranelift/filetests/filetests/isa/x64/f128const.clif +++ b/cranelift/filetests/filetests/isa/x64/f128const.clif @@ -66,4 +66,5 @@ block0(): ; addb %al, (%rax) ; addb %al, (%rax) ; addb %bh, %bh +; .byte 0x3f diff --git a/cranelift/filetests/filetests/isa/x64/fcvt-avx.clif b/cranelift/filetests/filetests/isa/x64/fcvt-avx.clif index 6548ad9f3cb2..e59edc0ed20c 100644 --- a/cranelift/filetests/filetests/isa/x64/fcvt-avx.clif +++ b/cranelift/filetests/filetests/isa/x64/fcvt-avx.clif @@ -154,6 +154,14 @@ block0(v0: i64x2): ; addb %al, (%rax) ; addb %al, (%rax) ; addb %al, (%rax) +; .byte 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 +; .byte 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 +; .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x43 +; .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x43 +; .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x45 +; .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x45 +; .byte 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x30, 0x45 +; .byte 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x30, 0x45 function %i64x2_to_f64x2(i64x2) -> f64x2 { block0(v0: i64x2): diff --git a/cranelift/filetests/filetests/isa/x64/fcvt.clif b/cranelift/filetests/filetests/isa/x64/fcvt.clif index a9ff6d48cc00..cca1d2801efd 100644 --- a/cranelift/filetests/filetests/isa/x64/fcvt.clif +++ b/cranelift/filetests/filetests/isa/x64/fcvt.clif @@ -372,6 +372,7 @@ block0(v0: i32x4): ; addb %al, (%rax) ; addb %al, (%rax) ; addb %dh, (%rax) +; .byte 0x43 function %f12(i32x4) -> f32x4 { block0(v0: i32x4): @@ -1195,6 +1196,14 @@ block0(v0: i64x2): ; addb %al, (%rax) ; addb %al, (%rax) ; addb %al, (%rax) +; .byte 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 +; .byte 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 +; .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x43 +; .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x43 +; .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x45 +; .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x45 +; .byte 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x30, 0x45 +; .byte 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x30, 0x45 function %i64x2_to_f64x2(i64x2) -> f64x2 { block0(v0: i64x2): diff --git a/cranelift/filetests/filetests/isa/x64/float-avx.clif b/cranelift/filetests/filetests/isa/x64/float-avx.clif index 37bbc6428afd..e00c30e765c7 100644 --- a/cranelift/filetests/filetests/isa/x64/float-avx.clif +++ b/cranelift/filetests/filetests/isa/x64/float-avx.clif @@ -592,6 +592,8 @@ block0(v0: f64x2): ; addb %al, (%rax) ; addb %al, (%rax) ; sarb $0xff, %bh +; .byte 0xff, 0xdf, 0x41, 0x00, 0x00, 0xc0, 0xff, 0xff +; .byte 0xff, 0xdf, 0x41 function %load_and_store_f32(i64, i64) { block0(v0: i64, v1: i64): diff --git a/cranelift/filetests/filetests/isa/x64/i128.clif b/cranelift/filetests/filetests/isa/x64/i128.clif index d607d884bff1..7340b8f5b127 100644 --- a/cranelift/filetests/filetests/isa/x64/i128.clif +++ b/cranelift/filetests/filetests/isa/x64/i128.clif @@ -2192,6 +2192,7 @@ block0(v0: i128, v1: i128): ; addb %ah, (%rax, %rax) ; addb %al, (%rax) ; addb %al, (%rax) +; .byte 0x00 function %uadd_overflow_as_i128(i64, i64) -> i64, i64 { block0(v0: i64, v1: i64): diff --git a/cranelift/filetests/filetests/isa/x64/immediates.clif b/cranelift/filetests/filetests/isa/x64/immediates.clif index ff73a774f753..d6b9a268fd63 100644 --- a/cranelift/filetests/filetests/isa/x64/immediates.clif +++ b/cranelift/filetests/filetests/isa/x64/immediates.clif @@ -59,4 +59,5 @@ block0(v0: i64, v1: i64): ; fstp %st(5) ; outb %al, %dx ; outb %al, %dx +; .byte 0xff, 0xff diff --git a/cranelift/filetests/filetests/isa/x64/mul.clif b/cranelift/filetests/filetests/isa/x64/mul.clif index cb18760e972c..a5e8aad798e2 100644 --- a/cranelift/filetests/filetests/isa/x64/mul.clif +++ b/cranelift/filetests/filetests/isa/x64/mul.clif @@ -698,4 +698,5 @@ block0: ; andb (%rdx), %ah ; andb (%rax), %al ; addb %al, (%rax) +; .byte 0x00 diff --git a/cranelift/filetests/filetests/isa/x64/narrowing.clif b/cranelift/filetests/filetests/isa/x64/narrowing.clif index edfdcb012af6..0264b1e14633 100644 --- a/cranelift/filetests/filetests/isa/x64/narrowing.clif +++ b/cranelift/filetests/filetests/isa/x64/narrowing.clif @@ -94,6 +94,8 @@ block0(v0: f64x2): ; addb %al, (%rax) ; addb %al, (%rax) ; addb %al, %al +; .byte 0xff, 0xff, 0xff, 0xdf, 0x41, 0x00, 0x00, 0xc0 +; .byte 0xff, 0xff, 0xff, 0xdf, 0x41 function %f4(i16x8, i16x8) -> i8x16 { block0(v0: i16x8, v1: i16x8): diff --git a/cranelift/filetests/filetests/isa/x64/select-issue-3744.clif b/cranelift/filetests/filetests/isa/x64/select-issue-3744.clif index 2d199dc67d37..50219660f795 100644 --- a/cranelift/filetests/filetests/isa/x64/select-issue-3744.clif +++ b/cranelift/filetests/filetests/isa/x64/select-issue-3744.clif @@ -40,4 +40,5 @@ block0(v0: f32, v1: f32): ; addb %al, (%rax) ; addb %al, (%rax) ; addb %al, (%rax) +; .byte 0x00 diff --git a/cranelift/filetests/filetests/isa/x64/shuffle-avx512.clif b/cranelift/filetests/filetests/isa/x64/shuffle-avx512.clif index e5c964e59b44..907cd6a2d072 100644 --- a/cranelift/filetests/filetests/isa/x64/shuffle-avx512.clif +++ b/cranelift/filetests/filetests/isa/x64/shuffle-avx512.clif @@ -40,6 +40,7 @@ block0(v0: i8x16, v1: i8x16): ; addb %al, (%rax) ; addb %al, (%rax) ; addb %al, (%rax) +; .byte 0x11 function %f3(i8x16, i8x16) -> i8x16 { block0(v0: i8x16, v1: i8x16): @@ -75,4 +76,5 @@ block0(v0: i8x16, v1: i8x16): ; addb %bl, (%rdi) ; sbbb (%rsi, %rax), %al ; orb $0xb, %al +; .byte 0x17, 0x0d, 0x18, 0x04, 0x02, 0x0f, 0x11, 0x05 diff --git a/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif b/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif index 38f879f37ab8..26ee82996cd4 100644 --- a/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif +++ b/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif @@ -596,6 +596,7 @@ block0(v0: i16x8, v1: i16x8): ; addb %al, (%rax) ; addb %al, -0x7fff8000(%rax) ; addb %al, -0x7fff8000(%rax) +; .byte 0x00, 0x80, 0x00, 0x80 function %i64x2_extmul_high_i32x4_s(i32x4, i32x4) -> i64x2 { block0(v0: i32x4, v1: i32x4): @@ -703,6 +704,7 @@ block0(v0: i32x4): ; addb %al, (%r8) ; addb %al, (%rax) ; addb %al, (%rax) +; .byte 0x30, 0x43 function %f32x4_add(f32x4, f32x4) -> f32x4 { block0(v0: f32x4, v1: f32x4): @@ -1298,6 +1300,7 @@ block0(v0: i16x8): ; addb %al, (%rcx) ; addb %al, (%rcx) ; addb %al, (%rcx) +; .byte 0x00 function %i8x16_splat(i8) -> i8x16 { block0(v0: i8): @@ -1370,6 +1373,10 @@ block0(v0: f64x2): ; addb %al, (%rax) ; addb %al, (%rax) ; loopne 0x33 +; .byte 0xff, 0xff, 0xef, 0x41, 0x00, 0x00, 0xe0, 0xff +; .byte 0xff, 0xff, 0xef, 0x41, 0x00, 0x00, 0x00, 0x00 +; .byte 0x00, 0x00, 0x30, 0x43, 0x00, 0x00, 0x00, 0x00 +; .byte 0x00, 0x00, 0x30, 0x43 function %i8x16_shl(i8x16, i32) -> i8x16 { block0(v0: i8x16, v1: i32): @@ -1410,6 +1417,22 @@ block0(v0: i8x16, v1: i32): ; addb %al, (%rax) ; addb %al, (%rax) ; addb %al, (%rax) +; .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +; .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +; .byte 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe +; .byte 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe +; .byte 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc +; .byte 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc +; .byte 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8 +; .byte 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8 +; .byte 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0 +; .byte 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0 +; .byte 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0 +; .byte 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0 +; .byte 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0 +; .byte 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0 +; .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 +; .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 function %i8x16_shl_imm(i8x16) -> i8x16 { block0(v0: i8x16): @@ -1443,6 +1466,8 @@ block0(v0: i8x16): ; addb %al, (%rax) ; addb %al, (%rax) ; addb %al, (%rax) +; .byte 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe +; .byte 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe function %i16x8_shl(i16x8, i32) -> i16x8 { block0(v0: i16x8, v1: i32): @@ -1648,6 +1673,22 @@ block0(v0: i8x16, v1: i32): ; addb %al, (%rax) ; addb %al, (%rax) ; addb %al, (%rax) +; .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +; .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +; .byte 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f +; .byte 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f +; .byte 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f +; .byte 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f +; .byte 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f +; .byte 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f +; .byte 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f +; .byte 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f +; .byte 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07 +; .byte 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07 +; .byte 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03 +; .byte 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03 +; .byte 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 +; .byte 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 function %i8x16_ushr_imm(i8x16) -> i8x16 { block0(v0: i8x16): diff --git a/cranelift/filetests/filetests/isa/x64/simd-bitselect.clif b/cranelift/filetests/filetests/isa/x64/simd-bitselect.clif index b9ebb8769783..9ba5b2b81f21 100644 --- a/cranelift/filetests/filetests/isa/x64/simd-bitselect.clif +++ b/cranelift/filetests/filetests/isa/x64/simd-bitselect.clif @@ -201,6 +201,7 @@ block0(v0: i16x8, v1: i16x8): ; addb %al, (%rax) ; incl (%rax) ; addb %al, (%rax) +; .byte 0xff, 0xff, 0xff, 0x00, 0xff, 0xff function %bad_const_mask(i8x16, i8x16) -> i8x16 { block0(v0: i8x16, v1: i8x16): @@ -250,4 +251,5 @@ block0(v0: i8x16, v1: i8x16): ; addb %bh, %bh ; addb %al, (%rax) ; addb %al, (%rax) +; .byte 0xff diff --git a/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif b/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif index 92703f5c35dc..5bd12ccaf8ed 100644 --- a/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif +++ b/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif @@ -362,6 +362,24 @@ block0(v0: i32): ; addb %al, (%rcx) ; addb (%rbx), %al ; addb $5, %al +; .byte 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d +; .byte 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +; .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +; .byte 0xff, 0xff, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe +; .byte 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe +; .byte 0xfe, 0xfe, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc +; .byte 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc +; .byte 0xfc, 0xfc, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8 +; .byte 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8 +; .byte 0xf8, 0xf8, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0 +; .byte 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0 +; .byte 0xf0, 0xf0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0 +; .byte 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0 +; .byte 0xe0, 0xe0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0 +; .byte 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0 +; .byte 0xc0, 0xc0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 +; .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 +; .byte 0x80, 0x80 function %ishl_i8x16_imm(i8x16) -> i8x16 { block0(v0: i8x16): @@ -395,6 +413,8 @@ block0(v0: i8x16): ; addb %al, (%rax) ; addb %al, (%rax) ; addb %al, (%rax) +; .byte 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0 +; .byte 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0 function %ishl_i16x8_imm(i16x8) -> i16x8 { block0(v0: i16x8): @@ -508,6 +528,10 @@ block0: ; addb %al, (%rcx) ; addb (%rbx), %al ; addb $5, %al +; .byte 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d +; .byte 0x0e, 0x0f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f +; .byte 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f +; .byte 0x7f, 0x7f function %ushr_i16x8_imm(i16x8) -> i16x8 { block0(v0: i16x8): @@ -639,6 +663,8 @@ block0(v0: i32): ; addb %al, (%rcx) ; addb (%rbx), %al ; addb $5, %al +; .byte 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d +; .byte 0x0e, 0x0f function %sshr_i8x16_imm(i8x16, i32) -> i8x16 { block0(v0: i8x16, v1: i32): @@ -926,4 +952,5 @@ block0(v0: i64x2, v1: i32): ; addb %al, (%rax) ; addb %al, (%rax) ; addb %al, (%rax) +; .byte 0x00, 0x80 diff --git a/cranelift/filetests/filetests/isa/x64/simd-lane-access-compile.clif b/cranelift/filetests/filetests/isa/x64/simd-lane-access-compile.clif index 0d1915f97895..394de63635dd 100644 --- a/cranelift/filetests/filetests/isa/x64/simd-lane-access-compile.clif +++ b/cranelift/filetests/filetests/isa/x64/simd-lane-access-compile.clif @@ -58,6 +58,7 @@ block0: ; addb %al, (%rax) ; addb $0x80, -0x7f7f7f80(%rax) ; addb $0x80, -0x7f7f7f80(%rax) +; .byte 0x80, 0x80, 0x01 function %shuffle_same_ssa_value() -> i8x16 { block0: @@ -146,6 +147,10 @@ block0: ; addb %al, (%rcx) ; addb (%rbx), %al ; addb $5, %al +; .byte 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d +; .byte 0x0e, 0x0f, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70 +; .byte 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70 +; .byte 0x70, 0x70 function %splat_i8(i8) -> i8x16 { block0(v0: i8): diff --git a/cranelift/filetests/filetests/isa/x64/simd-pairwise-add.clif b/cranelift/filetests/filetests/isa/x64/simd-pairwise-add.clif index 966c80a05323..3a3854ec71ad 100644 --- a/cranelift/filetests/filetests/isa/x64/simd-pairwise-add.clif +++ b/cranelift/filetests/filetests/isa/x64/simd-pairwise-add.clif @@ -84,6 +84,7 @@ block0(v0: i16x8): ; addb %al, (%rcx) ; addb %al, (%rcx) ; addb %al, (%rcx) +; .byte 0x00 function %fn3(i8x16) -> i16x8 { block0(v0: i8x16): diff --git a/cranelift/filetests/filetests/isa/x64/sqmul_round_sat.clif b/cranelift/filetests/filetests/isa/x64/sqmul_round_sat.clif index c970bf654ebe..4fbe8e94d426 100644 --- a/cranelift/filetests/filetests/isa/x64/sqmul_round_sat.clif +++ b/cranelift/filetests/filetests/isa/x64/sqmul_round_sat.clif @@ -34,4 +34,5 @@ block0(v0: i16x8, v1: i16x8): ; addb %al, (%rax) ; addb %al, -0x7fff8000(%rax) ; addb %al, -0x7fff8000(%rax) +; .byte 0x00, 0x80, 0x00, 0x80 diff --git a/cranelift/filetests/filetests/isa/x64/uunarrow.clif b/cranelift/filetests/filetests/isa/x64/uunarrow.clif index eb8392c32296..597909e8b93e 100644 --- a/cranelift/filetests/filetests/isa/x64/uunarrow.clif +++ b/cranelift/filetests/filetests/isa/x64/uunarrow.clif @@ -42,4 +42,8 @@ block0(v0: f64x2): ; addb %al, (%rax) ; addb %al, (%rax) ; addb %ah, %al +; .byte 0xff, 0xff, 0xff, 0xef, 0x41, 0x00, 0x00, 0xe0 +; .byte 0xff, 0xff, 0xff, 0xef, 0x41, 0x00, 0x00, 0x00 +; .byte 0x00, 0x00, 0x00, 0x30, 0x43, 0x00, 0x00, 0x00 +; .byte 0x00, 0x00, 0x00, 0x30, 0x43 From 866ca818e27d4177de50812a00d3a50fdb6b228d Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Wed, 15 Jul 2026 19:16:41 -0700 Subject: [PATCH 2/9] Add initial DSL bits for APX Extended-EVEX instructions --- cranelift/assembler-x64/meta/src/dsl.rs | 2 +- .../assembler-x64/meta/src/dsl/encoding.rs | 155 +++++++++++++++++- .../assembler-x64/meta/src/dsl/features.rs | 2 + .../meta/src/instructions/add.rs | 11 ++ cranelift/codegen/meta/src/isa/x86.rs | 9 + cranelift/codegen/src/isa/x64/inst/mod.rs | 4 + 6 files changed, 181 insertions(+), 2 deletions(-) diff --git a/cranelift/assembler-x64/meta/src/dsl.rs b/cranelift/assembler-x64/meta/src/dsl.rs index 69e7778d386c..293d914620d7 100644 --- a/cranelift/assembler-x64/meta/src/dsl.rs +++ b/cranelift/assembler-x64/meta/src/dsl.rs @@ -10,8 +10,8 @@ mod features; pub mod format; pub use custom::{Custom, Customization}; +pub use encoding::{ApxClass, Evex, Length, Vex, VexEscape, VexPrefix, evex, vex}; pub use encoding::{Encoding, ModRmKind, OpcodeMod}; -pub use encoding::{Evex, Length, Vex, VexEscape, VexPrefix, evex, vex}; pub use encoding::{ Group1Prefix, Group2Prefix, Group3Prefix, Group4Prefix, Opcodes, Prefixes, Rex, TupleType, rex, }; diff --git a/cranelift/assembler-x64/meta/src/dsl/encoding.rs b/cranelift/assembler-x64/meta/src/dsl/encoding.rs index c843e609d7eb..658c723bdc14 100644 --- a/cranelift/assembler-x64/meta/src/dsl/encoding.rs +++ b/cranelift/assembler-x64/meta/src/dsl/encoding.rs @@ -57,6 +57,9 @@ pub fn evex(length: Length, tuple_type: TupleType) -> Evex { modrm: None, imm: Imm::None, tuple_type, + apx: None, + nd: None, + nf: None, } } @@ -841,6 +844,10 @@ pub enum VexEscape { _0F, _0F3A, _0F38, + /// APX "opcode map 4"; only valid for APX legacy-GPR (Extended EVEX) + /// encodings, never for VEX. This is the map that enables promoting legacy + /// general-purpose-register instructions into the EVEX space. + _MAP4, } impl VexEscape { @@ -851,6 +858,7 @@ impl VexEscape { Self::_0F => 0b01, Self::_0F38 => 0b10, Self::_0F3A => 0b11, + Self::_MAP4 => 0b100, } } } @@ -861,6 +869,7 @@ impl fmt::Display for VexEscape { Self::_0F => write!(f, "0F"), Self::_0F3A => write!(f, "0F3A"), Self::_0F38 => write!(f, "0F38"), + Self::_MAP4 => write!(f, "MAP4"), } } } @@ -1247,6 +1256,30 @@ pub struct Evex { /// The "Tuple Type" corresponding to scaling of the 8-bit displacement /// parameter for memory operands. See [`TupleType`] for more information. pub tuple_type: TupleType, + /// The APX "Extended EVEX" class, when this instruction is an APX + /// promotion. + /// + /// Intel APX (Advanced Performance Extensions) reuses the `0x62` EVEX + /// identifier but does *not* define a single static payload layout. + /// Instead, the meaning of the payload bytes (`P0`, `P1`, `P2`) is + /// re-mapped depending on which class of instruction is being promoted (see + /// [`ApxClass`]). `None` denotes a standard (AVX-512) EVEX encoding; + /// `Some(_)` selects one of the APX layouts and enables addressing of the + /// extended general-purpose registers `R16`–`R31` ("EGPR"). + pub apx: Option, + /// The APX `ND` (New Data destination) bit. + /// + /// When set, a legacy destructive two-operand instruction is promoted into + /// a non-destructive three-operand form (the extra destination is encoded + /// in the `EVEX.vvvv` field). `None` when the bit is not part of this + /// encoding; only valid for [`ApxClass::LegacyGpr`]. + pub nd: Option, + /// The APX `NF` (No Flags) bit. + /// + /// When set, the status-flag writes that a legacy integer instruction would + /// otherwise perform are suppressed. `None` when the bit is not part of + /// this encoding; only valid for [`ApxClass::LegacyGpr`]. + pub nf: Option, } impl Evex { @@ -1310,6 +1343,27 @@ impl Evex { } } + /// Select the APX "opcode map 4", promoting a legacy general-purpose-register + /// instruction into the Extended EVEX space; equivalent to `.MAP4` in the + /// manual. + /// + /// This both sets the `mmm` map bits and marks the encoding as + /// [`ApxClass::LegacyGpr`], which in turn enables addressing of the extended + /// GPRs `R16`–`R31` ("EGPR") and permits the `ND`/`NF` bits. + /// + /// # Panics + /// + /// Panics if the map (`mmm`) or APX class has already been set. + pub fn map4(self) -> Self { + assert!(self.mmm.is_none()); + assert!(self.apx.is_none()); + Self { + mmm: Some(VexEscape::_MAP4), + apx: Some(ApxClass::LegacyGpr), + ..self + } + } + /// Set the `W` bit to `0`; equivalent to `.W0` in the manual. pub fn w0(self) -> Self { assert!(self.w.is_ignored()); @@ -1352,9 +1406,72 @@ impl Evex { } } + /// Mark this as an APX "Extended EVEX" encoding of the given [`ApxClass`]. + /// + /// This selects the APX payload layout to emit and enables addressing of + /// the extended general-purpose registers `R16`–`R31` ("EGPR"). + /// + /// # Panics + /// + /// Panics if an APX class has already been set. + pub fn apx(self, class: ApxClass) -> Self { + assert!(self.apx.is_none()); + Self { + apx: Some(class), + ..self + } + } + + /// Set the APX `ND` (New Data destination) bit; equivalent to `.ND` in the + /// manual. + /// + /// # Panics + /// + /// Panics if this is not an [`ApxClass::LegacyGpr`] encoding, or if the bit + /// has already been set. + pub fn nd(self) -> Self { + assert_eq!( + self.apx, + Some(ApxClass::LegacyGpr), + "the ND bit is only valid for APX legacy-GPR promotions" + ); + assert!(self.nd.is_none()); + Self { + nd: Some(true), + ..self + } + } + + /// Set the APX `NF` (No Flags) bit; equivalent to `.NF` in the manual. + /// + /// # Panics + /// + /// Panics if this is not an [`ApxClass::LegacyGpr`] encoding, or if the bit + /// has already been set. + pub fn nf(self) -> Self { + assert_eq!( + self.apx, + Some(ApxClass::LegacyGpr), + "the NF bit is only valid for APX legacy-GPR promotions" + ); + assert!(self.nf.is_none()); + Self { + nf: Some(true), + ..self + } + } + fn validate(&self, _operands: &[Operand]) { assert!(self.opcode != u8::MAX); assert!(self.mmm.is_some()); + // The `ND`/`NF` bits are only defined for APX legacy-GPR promotions. + if self.nd.is_some() || self.nf.is_some() { + assert_eq!( + self.apx, + Some(ApxClass::LegacyGpr), + "the ND/NF bits require an APX legacy-GPR encoding" + ); + } } /// Retrieve the digit extending the opcode, if available. @@ -1412,7 +1529,14 @@ impl fmt::Display for Evex { if let Some(mmmmm) = self.mmm { write!(f, ".{mmmmm}")?; } - write!(f, ".{} {:#04X}", self.w, self.opcode)?; + write!(f, ".{}", self.w)?; + if self.nd == Some(true) { + write!(f, ".ND")?; + } + if self.nf == Some(true) { + write!(f, ".NF")?; + } + write!(f, " {:#04X}", self.opcode)?; if let Some(modrm) = self.modrm { write!(f, " {modrm}")?; } @@ -1423,6 +1547,35 @@ impl fmt::Display for Evex { } } +/// The class of an APX "Extended EVEX" encoding. +/// +/// Intel APX does not define a single static Extended-EVEX layout. Although +/// every APX instruction still begins with the `0x62` EVEX identifier, the +/// meaning of the payload bytes (`P0`, `P1`, `P2`) is re-mapped depending on +/// the class of instruction being promoted. This enum selects which layout the +/// assembler should emit; all classes gain access to the extended +/// general-purpose registers `R16`–`R31` ("EGPR"). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum ApxClass { + /// Promotion of a legacy general-purpose-register (GPR) instruction using + /// extended opcode map 4. Only this class may set the `ND` and `NF` bits. + LegacyGpr, + /// Promotion of a legacy SSE / VEX vector instruction into the EVEX space. + Vector, + /// An existing AVX-512 (EVEX) instruction extended with APX register bits. + Avx512, +} + +impl fmt::Display for ApxClass { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::LegacyGpr => write!(f, "LegacyGpr"), + Self::Vector => write!(f, "Vector"), + Self::Avx512 => write!(f, "Avx512"), + } + } +} + /// Tuple Type definitions used in EVEX encodings. /// /// This enumeration corresponds to table 2-34 and 2-35 in the Intel manual. diff --git a/cranelift/assembler-x64/meta/src/dsl/features.rs b/cranelift/assembler-x64/meta/src/dsl/features.rs index d7cfdbe5ebfd..e9266d7db5b8 100644 --- a/cranelift/assembler-x64/meta/src/dsl/features.rs +++ b/cranelift/assembler-x64/meta/src/dsl/features.rs @@ -98,6 +98,7 @@ pub enum Feature { fma, avx_vnni, avx512vnni, + apx, } /// List all CPU features. @@ -131,6 +132,7 @@ pub const ALL_FEATURES: &[Feature] = &[ Feature::fma, Feature::avx_vnni, Feature::avx512vnni, + Feature::apx, ]; impl fmt::Display for Feature { diff --git a/cranelift/assembler-x64/meta/src/instructions/add.rs b/cranelift/assembler-x64/meta/src/instructions/add.rs index f5969061b053..309b4f9e8b1f 100644 --- a/cranelift/assembler-x64/meta/src/instructions/add.rs +++ b/cranelift/assembler-x64/meta/src/instructions/add.rs @@ -97,5 +97,16 @@ pub fn list() -> Vec { inst("vphaddw", fmt("B", [w(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x01).r(), (_64b | compat) & avx), inst("vphaddd", fmt("B", [w(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x02).r(), (_64b | compat) & avx), inst("vaddpd", fmt("C", [w(xmm1), r(xmm2), r(xmm_m128)]), evex(L128, Full)._66()._0f().w1().op(0x58).r(), (_64b | compat) & avx512vl), + // APX + // + // NOTE: The DSL now supports describing APX "Extended EVEX" encodings + // via `evex(..).map4().nd()/.nf()` (see `dsl::encoding`). Emitting a + // real APX instruction additionally requires format/emitter codegen + // support (a three-operand ND format, EGPR register bits, and the + // MAP4/ND/NF payload emission in `generate::format`), which is not yet + // wired up. Example of the intended DSL usage once codegen lands: + // + // inst("addq", fmt("RVM", [w(r64a), r(r64b), r(rm64)]), + // evex(L128, Full).map4().w0().nd().op(0x01).r(), _64b), ] } diff --git a/cranelift/codegen/meta/src/isa/x86.rs b/cranelift/codegen/meta/src/isa/x86.rs index d72be07de2c4..836338027f40 100644 --- a/cranelift/codegen/meta/src/isa/x86.rs +++ b/cranelift/codegen/meta/src/isa/x86.rs @@ -95,6 +95,15 @@ pub(crate) fn define() -> TargetIsa { "AVX512VNNI: CPUID.07H:ECX.AVX512_VNNI[bit 11]", false, ); + // Registered so `target x86_64 has_apx` parses; not yet referenced by any + // preset because no shipping microarchitecture ships APX today. Drop the + // leading underscore once a preset needs it. + let _has_apx = settings.add_bool( + "has_apx", + "Has support for APX.", + "APX_F: CPUID.(EAX=07H,ECX=1H):EDX.APX_F[bit 21]", + false, + ); let has_popcnt = settings.add_bool( "has_popcnt", "Has support for POPCNT.", diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 8134eb5dc6e7..a2e6306c94fd 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -1595,6 +1595,10 @@ impl asm::AvailableFeatures for &EmitInfo { fn avx512vnni(&self) -> bool { self.isa_flags.has_avx512vnni() } + + fn apx(&self) -> bool { + self.isa_flags.has_apx() + } } impl MachInstEmit for Inst { From ceb7961a9097efde35c751afda4141b1a3b7270a Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Sun, 19 Jul 2026 21:38:03 -0700 Subject: [PATCH 3/9] Emit APX Extended-EVEX legacy encodings in the x64 assembler Add the runtime EVEX map-4 prefix builder (ND/NF/EGPR bits per APX spec rev 8, Fig. 3.3), wire it into the meta code generator, and enable the NDD `addq` instruction. Verified with a byte-exact encoding test (62 F4 F4 18 01 C2). Instruction selection (ISLE lowering) and EGPR register allocation are not part of this change; the `.clif` filetest cannot select these instructions yet. --- .../assembler-x64/meta/src/generate/format.rs | 54 +++++++--- .../meta/src/instructions/add.rs | 16 +-- cranelift/assembler-x64/src/evex.rs | 101 ++++++++++++++++++ cranelift/assembler-x64/src/fuzz.rs | 31 ++++++ 4 files changed, 175 insertions(+), 27 deletions(-) diff --git a/cranelift/assembler-x64/meta/src/generate/format.rs b/cranelift/assembler-x64/meta/src/generate/format.rs index e1e4a8bb3bfb..470b23dc91fb 100644 --- a/cranelift/assembler-x64/meta/src/generate/format.rs +++ b/cranelift/assembler-x64/meta/src/generate/format.rs @@ -243,7 +243,7 @@ impl dsl::Format { fmtln!(f, "let w = {};", vex.w.as_bool()); let bits = "len, pp, mmmmm, w"; - self.generate_vex_or_evex_prefix(f, "VexPrefix", &bits, vex.is4, None, || { + self.generate_vex_or_evex_prefix(f, "VexPrefix", &bits, vex.is4, None, "two_op", "three_op", || { vex.unwrap_digit() }) } @@ -251,16 +251,36 @@ impl dsl::Format { fn generate_evex_prefix(&self, f: &mut Formatter, evex: &dsl::Evex) -> ModRmStyle { f.empty_line(); f.comment("Emit EVEX prefix."); - let ll = evex.length.evex_bits(); - fmtln!(f, "let ll = {ll:#04b};"); + + // Intel APX promotes legacy GPR instructions into "EVEX map 4" using a + // re-purposed payload layout (see `EvexPrefix::legacy` in the runtime + // assembler and section 3.1.2.3.1 of the APX spec). In that case we + // emit the ND/NF bits and select the `legacy_*` constructors instead of + // the AVX-512 ones. + let apx_legacy = matches!(evex.apx, Some(dsl::ApxClass::LegacyGpr)); + fmtln!(f, "let pp = {:#04b};", evex.pp.map_or(0b00, |pp| pp.bits())); fmtln!(f, "let mmm = {:#07b};", evex.mmm.unwrap().bits()); fmtln!(f, "let w = {};", evex.w.as_bool()); - // NB: when bcast is supported in the future the `evex_scaling` - // calculation for `Full` and `Half` below need to be updated. + + let (bits, two_op, three_op); + if apx_legacy { + fmtln!(f, "let nd = {};", evex.nd == Some(true)); + fmtln!(f, "let nf = {};", evex.nf == Some(true)); + bits = String::from("pp, mmm, w, nd, nf"); + two_op = "legacy_two_op"; + three_op = "legacy_three_op"; + } else { + let ll = evex.length.evex_bits(); + fmtln!(f, "let ll = {ll:#04b};"); + // NB: when bcast is supported in the future the `evex_scaling` + // calculation for `Full` and `Half` below need to be updated. + fmtln!(f, "let bcast = false;"); + bits = String::from("ll, pp, mmm, w, bcast"); + two_op = "two_op"; + three_op = "three_op"; + } let bcast = false; - fmtln!(f, "let bcast = {bcast};"); - let bits = format!("ll, pp, mmm, w, bcast"); let is4 = false; let length_bytes = match evex.length { @@ -303,7 +323,7 @@ impl dsl::Format { }, }); - self.generate_vex_or_evex_prefix(f, "EvexPrefix", &bits, is4, evex_scaling, || { + self.generate_vex_or_evex_prefix(f, "EvexPrefix", &bits, is4, evex_scaling, two_op, three_op, || { evex.unwrap_digit() }) } @@ -318,6 +338,8 @@ impl dsl::Format { bits: &str, is4: bool, evex_scaling: Option, + two_op: &str, + three_op: &str, unwrap_digit: impl Fn() -> Option, ) -> ModRmStyle { use dsl::OperandKind::{FixedReg, Imm, Mem, Reg, RegMem}; @@ -330,7 +352,7 @@ impl dsl::Format { fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); fmtln!( f, - "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});" + "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});" ); ModRmStyle::Reg { reg: ModRmReg::Reg(*reg), @@ -347,7 +369,7 @@ impl dsl::Format { fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); fmtln!( f, - "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});" + "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});" ); ModRmStyle::RegMem { reg: ModRmReg::Reg(*reg), @@ -362,7 +384,7 @@ impl dsl::Format { fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); fmtln!( f, - "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});" + "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});" ); ModRmStyle::RegMemIs4 { reg: ModRmReg::Reg(*reg), @@ -382,7 +404,7 @@ impl dsl::Format { fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); fmtln!( f, - "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});" + "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});" ); ModRmStyle::RegMem { reg: ModRmReg::Digit(digit), @@ -395,7 +417,7 @@ impl dsl::Format { let reg = reg_or_vvvv; fmtln!(f, "let reg = self.{reg}.enc();"); fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); - fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});"); + fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});"); ModRmStyle::RegMem { reg: ModRmReg::Reg(*reg), rm: *rm, @@ -413,7 +435,7 @@ impl dsl::Format { fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); fmtln!( f, - "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});" + "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});" ); ModRmStyle::Reg { reg: ModRmReg::Digit(digit), @@ -425,7 +447,7 @@ impl dsl::Format { let reg = reg_or_vvvv; fmtln!(f, "let reg = self.{reg}.enc();"); fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); - fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});"); + fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});"); ModRmStyle::Reg { reg: ModRmReg::Reg(*reg), rm: *rm, @@ -437,7 +459,7 @@ impl dsl::Format { assert!(!is4); fmtln!(f, "let reg = self.{reg}.enc();"); fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); - fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});"); + fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});"); ModRmStyle::RegMem { reg: ModRmReg::Reg(*reg), rm: *rm, diff --git a/cranelift/assembler-x64/meta/src/instructions/add.rs b/cranelift/assembler-x64/meta/src/instructions/add.rs index 309b4f9e8b1f..7abadba648c0 100644 --- a/cranelift/assembler-x64/meta/src/instructions/add.rs +++ b/cranelift/assembler-x64/meta/src/instructions/add.rs @@ -97,16 +97,10 @@ pub fn list() -> Vec { inst("vphaddw", fmt("B", [w(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x01).r(), (_64b | compat) & avx), inst("vphaddd", fmt("B", [w(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x02).r(), (_64b | compat) & avx), inst("vaddpd", fmt("C", [w(xmm1), r(xmm2), r(xmm_m128)]), evex(L128, Full)._66()._0f().w1().op(0x58).r(), (_64b | compat) & avx512vl), - // APX - // - // NOTE: The DSL now supports describing APX "Extended EVEX" encodings - // via `evex(..).map4().nd()/.nf()` (see `dsl::encoding`). Emitting a - // real APX instruction additionally requires format/emitter codegen - // support (a three-operand ND format, EGPR register bits, and the - // MAP4/ND/NF payload emission in `generate::format`), which is not yet - // wired up. Example of the intended DSL usage once codegen lands: - // - // inst("addq", fmt("RVM", [w(r64a), r(r64b), r(rm64)]), - // evex(L128, Full).map4().w0().nd().op(0x01).r(), _64b), + // APX: NDD (new data destination) form of `ADD`, promoted into EVEX + // "map 4" via the extended-EVEX prefix. Operands are `[NDD dest in + // reg, source in vvvv, r/m source]`. `addq` is 64-bit so the `W` bit is + // set (`.w1()`); use `.w0()` for the 32-bit `addl` form. + inst("addq", fmt("RVM", [w(r64a), r(r64b), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x01).r(), _64b), ] } diff --git a/cranelift/assembler-x64/src/evex.rs b/cranelift/assembler-x64/src/evex.rs index bf89dd13bdc1..2734fae7ac22 100644 --- a/cranelift/assembler-x64/src/evex.rs +++ b/cranelift/assembler-x64/src/evex.rs @@ -92,6 +92,107 @@ impl EvexPrefix { EvexPrefix::new(reg, vvvv, (b, x), ll, pp, mmm, w, broadcast) } + // --------------------------------------------------------------------- + // Intel APX "Extended EVEX" prefix for promoted *legacy* GPR instructions + // (EVEX map 4). See the Intel APX Architecture Specification (rev 8), + // section 3.1.2.3.1 "EVEX Extension of Legacy Instructions", Figure 3.3. + // + // The four bytes still begin with `0x62`, but several payload bits are + // re-purposed relative to the AVX-512 layout above: + // + // ┌────┬────┬────┬────┬────┬────┬────┬────┐ + // Byte 1: │ R3 │ X3 │ B3 │ R4 │ B4 │ 1 │ 0 │ 0 │ (map id = 4) + // ├────┼────┼────┼────┼────┼────┼────┼────┤ + // Byte 2: │ W │ V3 │ V2 │ V1 │ V0 │ U │ p │ p │ (U = ~X4) + // ├────┼────┼────┼────┼────┼────┼────┼────┤ + // Byte 3: │ 0 │ 0 │ 0 │ ND │ V4 │ NF │ 0 │ 0 │ + // └────┴────┴────┴────┴────┴────┴────┴────┘ + // + // The "underlined" fields (`R3`, `X3`, `B3`, `R4`, the `vvvv` bits and `V4`) + // are stored inverted, exactly as in the AVX-512 layout. `B4` and `X4` are + // newly repurposed reserved bits: `B4` uses *true* polarity (fixed value 0) + // and `X4` is carried inverted in the `U` bit (`EVEX.X4 = ~EVEX.U`), so a + // register-form instruction (ModRM.Mod = 3, no index) has `U = 1`. + + /// Construct the extended-EVEX (APX map 4) prefix for a legacy GPR + /// instruction. + /// + /// `reg` is the ModRM.reg register, `vvvv` is the `V` register identifier + /// (the NDD register when `nd` is set), and `(b, x)` are the ModRM.r/m base + /// and (optional) SIB index registers. `nd`/`nf` select the New Data + /// destination and No Flags bits respectively. + pub fn legacy( + reg: u8, + vvvv: u8, + (b, x): (Option, Option), + pp: u8, + mmm: u8, + w: bool, + nd: bool, + nf: bool, + ) -> Self { + let base = b.unwrap_or(0); + let index = x.unwrap_or(0); + + // byte1 (P0) + let r3 = invert_top_bit(reg); + let x3 = invert_top_bit(index); + let b3 = invert_top_bit(base); + let r4 = invert_top_bit(reg >> 1); + let b4 = (base >> 4) & 1; // true polarity + debug_assert!(mmm <= 0b111); + let byte1 = r3 << 7 | x3 << 6 | b3 << 5 | r4 << 4 | b4 << 3 | mmm; + + // byte2 (P1) + debug_assert!(vvvv <= 0b11111); + debug_assert!(pp <= 0b11); + let vvvv_value = !vvvv & 0b1111; + // `EVEX.X4 = ~EVEX.U`; with no index register X4 = 0 so U = 1. + let x4 = (index >> 4) & 1; + let u = (!x4) & 1; + let byte2 = (w as u8) << 7 | vvvv_value << 3 | u << 2 | (pp & 0b11); + + // byte3 (P2) + let v_prime = invert_top_bit(vvvv >> 1); // V4, inverted + let byte3 = (nd as u8) << 4 | v_prime << 3 | (nf as u8) << 2; + + Self { + byte1, + byte2, + byte3, + } + } + + /// Construct the extended-EVEX (APX map 4) prefix for a two-operand legacy + /// GPR instruction (no NDD); the `V` register identifier is unused. + #[allow(dead_code, reason = "not all APX legacy forms are emitted yet")] + pub fn legacy_two_op( + reg: u8, + (b, x): (Option, Option), + pp: u8, + mmm: u8, + w: bool, + nd: bool, + nf: bool, + ) -> Self { + EvexPrefix::legacy(reg, 0, (b, x), pp, mmm, w, nd, nf) + } + + /// Construct the extended-EVEX (APX map 4) prefix for a three-operand + /// legacy GPR instruction; `vvvv` carries the NDD register. + pub fn legacy_three_op( + reg: u8, + vvvv: u8, + (b, x): (Option, Option), + pp: u8, + mmm: u8, + w: bool, + nd: bool, + nf: bool, + ) -> Self { + EvexPrefix::legacy(reg, vvvv, (b, x), pp, mmm, w, nd, nf) + } + pub(crate) fn encode(&self, sink: &mut impl CodeSink) { sink.put1(0x62); sink.put1(self.byte1); diff --git a/cranelift/assembler-x64/src/fuzz.rs b/cranelift/assembler-x64/src/fuzz.rs index 3efe0acfc472..b5bdbf92909b 100644 --- a/cranelift/assembler-x64/src/fuzz.rs +++ b/cranelift/assembler-x64/src/fuzz.rs @@ -589,4 +589,35 @@ mod test { }) .budget_ms(1_000); } + + /// Byte-level encoding check for the APX NDD (new-data-destination) form of + /// `ADD`, promoted into EVEX "map 4" via the extended-EVEX prefix. + /// + /// We assert the exact bytes rather than round-tripping through Capstone + /// because the bundled disassembler does not yet understand APX. The + /// expected sequence for `addq` with ModRM.reg = rax (0), vvvv = rcx (1), + /// r/m = rdx (2), `W = 1`, `ND = 1`, `NF = 0` is: + /// + /// ```text + /// 62 F4 F4 18 01 C2 + /// ^^ ^^ ^^ ^^ ^^ ^^ + /// | | | | | └ ModRM: mod=11 reg=rax r/m=rdx + /// | | | | └ opcode 0x01 (ADD r/m, reg) + /// | | | └ P2: ND=1 (bit4), V4=1 (bit3, inverted), NF=0 + /// | | └ P1: W=1, vvvv=~rcx=1110, U=1, pp=00 + /// | └ P0: R3 X3 B3 R4 = 1111, B4=0, map=100 (map 4) + /// └ EVEX identifier + /// ``` + /// + /// NOTE: with `ND = 1` the architectural destination is the `vvvv` register + /// (here rcx), while the DSL currently places the `w()` operand in + /// ModRM.reg. Aligning the written operand with `vvvv` is the natural next + /// step for full NDD support. + #[test] + fn apx_addq_rvm_ndd_encoding() { + use crate::inst::addq_rvm; + let inst = addq_rvm::::new(FuzzReg::new(0), FuzzReg::new(1), FuzzReg::new(2)); + let assembled = assemble(&inst.into()); + assert_eq!(pretty_print_hexadecimal(&assembled), "62F4F41801C2"); + } } From c60018bdcd2dbbb41cd43a8cfd4de9d4533bc459 Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Sun, 19 Jul 2026 23:07:28 -0700 Subject: [PATCH 4/9] Wire APX NDD add into x64 instruction selection Add a `use_apx` ISLE predicate and lower 64-bit `iadd` to the NDD form of `addq` (EVEX map 4) when the `has_apx` target flag is enabled. Gate the instruction on the `apx` assembler feature and align the NDD destination with the EVEX `vvvv` operand so the result is written to a fresh register without clobbering either input. Add filetests exercising the new lowering and the `has_apx` flag parsing. --- .../meta/src/instructions/add.rs | 10 ++++--- cranelift/assembler-x64/src/fuzz.rs | 28 ++++++++++--------- cranelift/codegen/src/isa/x64/inst.isle | 3 ++ cranelift/codegen/src/isa/x64/lower.isle | 9 ++++++ cranelift/codegen/src/isa/x64/lower/isle.rs | 5 ++++ .../filetests/filetests/isa/x64/apx-add.clif | 26 +++++++++++++++++ 6 files changed, 64 insertions(+), 17 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/x64/apx-add.clif diff --git a/cranelift/assembler-x64/meta/src/instructions/add.rs b/cranelift/assembler-x64/meta/src/instructions/add.rs index 7abadba648c0..f4b6b4fb296f 100644 --- a/cranelift/assembler-x64/meta/src/instructions/add.rs +++ b/cranelift/assembler-x64/meta/src/instructions/add.rs @@ -98,9 +98,11 @@ pub fn list() -> Vec { inst("vphaddd", fmt("B", [w(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x02).r(), (_64b | compat) & avx), inst("vaddpd", fmt("C", [w(xmm1), r(xmm2), r(xmm_m128)]), evex(L128, Full)._66()._0f().w1().op(0x58).r(), (_64b | compat) & avx512vl), // APX: NDD (new data destination) form of `ADD`, promoted into EVEX - // "map 4" via the extended-EVEX prefix. Operands are `[NDD dest in - // reg, source in vvvv, r/m source]`. `addq` is 64-bit so the `W` bit is - // set (`.w1()`); use `.w0()` for the 32-bit `addl` form. - inst("addq", fmt("RVM", [w(r64a), r(r64b), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x01).r(), _64b), + // "map 4" via the extended-EVEX prefix. With `ND = 1` the architectural + // destination is the `vvvv`-encoded register, so the written operand is + // placed in the `V` slot (the middle operand of the `RVM` format) while + // the two sources occupy ModRM.reg and ModRM.rm. `addq` is 64-bit so + // the `W` bit is set (`.w1()`); use `.w0()` for the 32-bit `addl` form. + inst("addq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x01).r(), _64b & apx), ] } diff --git a/cranelift/assembler-x64/src/fuzz.rs b/cranelift/assembler-x64/src/fuzz.rs index b5bdbf92909b..f10e744e2584 100644 --- a/cranelift/assembler-x64/src/fuzz.rs +++ b/cranelift/assembler-x64/src/fuzz.rs @@ -594,30 +594,32 @@ mod test { /// `ADD`, promoted into EVEX "map 4" via the extended-EVEX prefix. /// /// We assert the exact bytes rather than round-tripping through Capstone - /// because the bundled disassembler does not yet understand APX. The - /// expected sequence for `addq` with ModRM.reg = rax (0), vvvv = rcx (1), - /// r/m = rdx (2), `W = 1`, `ND = 1`, `NF = 0` is: + /// because the bundled disassembler does not yet understand APX. With + /// `ND = 1` the architectural destination is the `vvvv`-encoded register, so + /// the `RVM` operands are `[ModRM.reg source, vvvv destination, ModRM.rm + /// source]`. For `addq %rax, %rcx, %rdx` (destination `%rax` = 0 in `vvvv`, + /// source `%rcx` = 1 in ModRM.reg, source `%rdx` = 2 in ModRM.rm), `W = 1`, + /// `ND = 1`, `NF = 0`, the expected sequence is: /// /// ```text - /// 62 F4 F4 18 01 C2 + /// 62 F4 FC 18 01 CA /// ^^ ^^ ^^ ^^ ^^ ^^ - /// | | | | | └ ModRM: mod=11 reg=rax r/m=rdx + /// | | | | | └ ModRM: mod=11 reg=rcx r/m=rdx /// | | | | └ opcode 0x01 (ADD r/m, reg) /// | | | └ P2: ND=1 (bit4), V4=1 (bit3, inverted), NF=0 - /// | | └ P1: W=1, vvvv=~rcx=1110, U=1, pp=00 + /// | | └ P1: W=1, vvvv=~rax=1111, U=1, pp=00 /// | └ P0: R3 X3 B3 R4 = 1111, B4=0, map=100 (map 4) /// └ EVEX identifier /// ``` - /// - /// NOTE: with `ND = 1` the architectural destination is the `vvvv` register - /// (here rcx), while the DSL currently places the `w()` operand in - /// ModRM.reg. Aligning the written operand with `vvvv` is the natural next - /// step for full NDD support. #[test] fn apx_addq_rvm_ndd_encoding() { use crate::inst::addq_rvm; - let inst = addq_rvm::::new(FuzzReg::new(0), FuzzReg::new(1), FuzzReg::new(2)); + // Format is `RVM` = [ModRM.reg source, vvvv destination, ModRM.rm + // source], so the constructor arguments are (reg source = %rcx, + // destination = %rax, r/m source = %rdx). + let inst = + addq_rvm::::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2)); let assembled = assemble(&inst.into()); - assert_eq!(pretty_print_hexadecimal(&assembled), "62F4F41801C2"); + assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1801CA"); } } diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 57639f6ff06f..ca321a0a6089 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -1336,6 +1336,9 @@ (decl pure use_avx2 () bool) (extern constructor use_avx2 use_avx2) +(decl pure use_apx () bool) +(extern constructor use_apx use_apx) + (decl pure has_cmpxchg16b () bool) (extern constructor has_cmpxchg16b has_cmpxchg16b) diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 8739c6fb6c27..5f99cf7ab020 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -94,6 +94,15 @@ (rule iadd_base_case_32_or_64_lea -5 (lower (iadd (ty_32_or_64 ty) x y)) (x64_lea ty (to_amode_add (mem_flags_trusted_data) x y (zero_offset)))) +;; APX: when the `has_apx` feature is enabled, use the NDD (new-data- +;; destination) form of `add` for 64-bit adds. Unlike the legacy two-operand +;; `add`, the NDD form writes its result to a fresh register encoded in the +;; EVEX `vvvv` field, so the register allocator does not need to insert a move +;; to preserve either input. +(rule iadd_apx_ndd 1 (lower (iadd $I64 x y)) + (if-let true (use_apx)) + (x64_addq_rvm x y)) + ;; Higher-priority cases than the previous two where a load can be sunk into ;; the add instruction itself. Note that both operands are tested for ;; sink-ability since addition is commutative diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 8c04e7f89369..c5b33e162304 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -492,6 +492,11 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> { self.backend.x64_flags.has_avx512vbmi() } + #[inline] + fn use_apx(&mut self) -> bool { + self.backend.x64_flags.has_apx() + } + #[inline] fn has_lzcnt(&mut self) -> bool { self.backend.x64_flags.has_lzcnt() diff --git a/cranelift/filetests/filetests/isa/x64/apx-add.clif b/cranelift/filetests/filetests/isa/x64/apx-add.clif new file mode 100644 index 000000000000..e71681fce2a2 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/apx-add.clif @@ -0,0 +1,26 @@ +test compile precise-output +target x86_64 has_apx + +function %add_i64(i64, i64) -> i64 { +block0(v0: i64, v1: i64): + v2 = iadd v0, v1 + return v2 +} + +; VCode: +; pushq %rbp +; movq %rsp, %rbp +; block0: +; addq %rsi, %rax, %rdi +; movq %rbp, %rsp +; popq %rbp +; retq +; +; Disassembled: +; block0: ; offset 0x0 +; pushq %rbp +; movq %rsp, %rbp +; block1: ; offset 0x4 +; .byte 0x62, 0xf4, 0xfc, 0x18, 0x01, 0xfe, 0x48, 0x89 +; .byte 0xec, 0x5d, 0xc3 + From 89b8c97ac8ce6a1876ab58e5057c61a5312d89f5 Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Thu, 13 Aug 2026 15:58:02 -0700 Subject: [PATCH 5/9] Print the APX NDD destination operand last The assembler's generated `Display` impl derives AT&T operand order by reversing the DSL's Intel-style order. That works when the destination is the ModRM.reg operand, but APX "new data destination" (NDD) forms encode their destination in `vvvv`, which the DSL must list in the middle so that the positional slot assignment in `generate_vex_or_evex_prefix` maps operands to `reg`/`vvvv`/`rm` correctly. Reversing therefore left the destination in the middle: `addq` printed as `addq %rsi, %rax, %rdi` where both AT&T convention and XED expect the destination last. Identify these forms from the `ND` bit rather than from the operand shape. `[Reg, Reg, RegMem]` is also used by ordinary VEX instructions, and `[Reg, RegMem, Reg]` is used by the BMI2 `bzhi`/`sarx`/`shlx`/`shrx` family with the slots assigned the other way around, so the shape alone cannot identify NDD. When `ND = 1`, sources keep their DSL order and the destination is printed last. Any future NDD instruction then gets the correct order without a per-instruction `Display` customization. Only printing changes here; the emitted encodings are untouched. --- .../assembler-x64/meta/src/dsl/encoding.rs | 7 +++ .../assembler-x64/meta/src/generate/format.rs | 47 +++++++++++++------ .../assembler-x64/meta/src/generate/inst.rs | 5 +- .../filetests/filetests/isa/x64/apx-add.clif | 2 +- 4 files changed, 43 insertions(+), 18 deletions(-) diff --git a/cranelift/assembler-x64/meta/src/dsl/encoding.rs b/cranelift/assembler-x64/meta/src/dsl/encoding.rs index 658c723bdc14..2d254be9c46e 100644 --- a/cranelift/assembler-x64/meta/src/dsl/encoding.rs +++ b/cranelift/assembler-x64/meta/src/dsl/encoding.rs @@ -89,6 +89,13 @@ impl Encoding { Encoding::Evex(evex) => evex.opcode, } } + + /// Return whether this encoding sets the APX `ND` ("new data destination") + /// bit, meaning the architectural destination is the `vvvv`-encoded + /// register rather than an operand named by ModRM. + pub fn is_nd(&self) -> bool { + matches!(self, Encoding::Evex(evex) if evex.nd == Some(true)) + } } impl fmt::Display for Encoding { diff --git a/cranelift/assembler-x64/meta/src/generate/format.rs b/cranelift/assembler-x64/meta/src/generate/format.rs index 470b23dc91fb..fa4775bb00d9 100644 --- a/cranelift/assembler-x64/meta/src/generate/format.rs +++ b/cranelift/assembler-x64/meta/src/generate/format.rs @@ -54,29 +54,46 @@ impl dsl::Format { /// once Cranelift has switched to using this assembler predominantly /// (TODO). #[must_use] - pub(crate) fn generate_att_style_operands(&self) -> String { - let ordered_ops: Vec<_> = self - .operands - .iter() - .filter(|o| !o.implicit) - .rev() - .map(|o| format!("{{{}}}", o.location)) - .collect(); - ordered_ops.join(", ") + pub(crate) fn generate_att_style_operands(&self, nd: bool) -> String { + self.ordered_operands(nd, false) } /// Like [`Self::generate_att_style_operands`], but omits the fixed `%xmm0` /// mask operand, which XED leaves implicit. #[must_use] - pub(crate) fn generate_xed_style_operands(&self) -> String { - let ordered_ops: Vec<_> = self + pub(crate) fn generate_xed_style_operands(&self, nd: bool) -> String { + self.ordered_operands(nd, true) + } + + /// Shared operand ordering for AT&T-style printing. + /// + /// Normally this is just the reverse of the DSL's Intel-style order. APX + /// "new data destination" (NDD) forms are the exception: their + /// architectural destination is the `vvvv`-encoded register, which the DSL + /// must list in the middle so that the positional slot assignment in + /// `generate_vex_or_evex_prefix` maps operands to `reg`/`vvvv`/`rm` + /// correctly. Blindly reversing would therefore print the destination in + /// the middle. Instead, for `ND = 1` the sources keep their DSL order and + /// the destination is printed last, which matches both AT&T convention and + /// the XED disassembly used as a fuzzing oracle. + #[must_use] + fn ordered_operands(&self, nd: bool, xed_style: bool) -> String { + let ops = self .operands .iter() - .filter(|o| !o.implicit && o.location != dsl::Location::xmm0) - .rev() + .filter(|o| !o.implicit && !(xed_style && o.location == dsl::Location::xmm0)); + let ordered: Vec<&dsl::Operand> = if nd { + let (dst, srcs): (Vec<_>, Vec<_>) = + ops.partition(|o| matches!(o.mutability, dsl::Mutability::Write)); + srcs.into_iter().chain(dst).collect() + } else { + ops.rev().collect() + }; + ordered + .into_iter() .map(|o| format!("{{{}}}", o.location)) - .collect(); - ordered_ops.join(", ") + .collect::>() + .join(", ") } #[must_use] diff --git a/cranelift/assembler-x64/meta/src/generate/inst.rs b/cranelift/assembler-x64/meta/src/generate/inst.rs index a2eb137356ef..71ec243abd0a 100644 --- a/cranelift/assembler-x64/meta/src/generate/inst.rs +++ b/cranelift/assembler-x64/meta/src/generate/inst.rs @@ -324,8 +324,9 @@ impl dsl::Inst { None => fmtln!(f, "let {location} = {to_string};"), } } - let ordered_ops = self.format.generate_att_style_operands(); - let xed_ops = self.format.generate_xed_style_operands(); + let nd = self.encoding.is_nd(); + let ordered_ops = self.format.generate_att_style_operands(nd); + let xed_ops = self.format.generate_xed_style_operands(nd); let mut implicit_ops = self.format.generate_implicit_operands(); if self.has_trap { fmtln!(f, "let trap = self.trap;"); diff --git a/cranelift/filetests/filetests/isa/x64/apx-add.clif b/cranelift/filetests/filetests/isa/x64/apx-add.clif index e71681fce2a2..94fcbfd18071 100644 --- a/cranelift/filetests/filetests/isa/x64/apx-add.clif +++ b/cranelift/filetests/filetests/isa/x64/apx-add.clif @@ -11,7 +11,7 @@ block0(v0: i64, v1: i64): ; pushq %rbp ; movq %rsp, %rbp ; block0: -; addq %rsi, %rax, %rdi +; addq %rdi, %rsi, %rax ; movq %rbp, %rsp ; popq %rbp ; retq From 6dfbc80cf73748cdd540e0b4f64d4db3eecba27c Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Thu, 13 Aug 2026 15:58:14 -0700 Subject: [PATCH 6/9] Skip capstone in the fuzz oracle for APX instructions The bundled capstone build cannot decode the APX extended-EVEX ("map 4") encodings at all, returning zero instructions, which tripped the "not a single instruction" assertion in the roundtrip oracle. Skip them the same way AVX-VNNI instructions are already skipped. APX stays in `ALL_FEATURES` so that the XED oracle, which does understand map 4, keeps exercising these instructions rather than losing coverage. --- cranelift/assembler-x64/src/fuzz.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cranelift/assembler-x64/src/fuzz.rs b/cranelift/assembler-x64/src/fuzz.rs index f10e744e2584..9eca241194ed 100644 --- a/cranelift/assembler-x64/src/fuzz.rs +++ b/cranelift/assembler-x64/src/fuzz.rs @@ -34,6 +34,13 @@ pub fn roundtrip(inst: &Inst) { return; } + // Likewise, capstone cannot decode the APX extended-EVEX ("map 4") + // encodings at all, returning zero instructions. These are instead checked + // against XED by `roundtrip_xed`, which does understand map 4. + if features_mention(inst.features(), Feature::apx) { + return; + } + roundtrip_with( inst, "capstone", From 3fb238a361dbe1debad26443c72dbc9ec17c68fb Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Fri, 21 Aug 2026 14:52:11 -0700 Subject: [PATCH 7/9] Don't scale disp8 for APX map-4 encodings EVEX normally stores an 8-bit displacement as a multiple of a tuple-derived factor N, so a 128-bit `Full` tuple divides the byte offset by 16. APX promotes legacy GPR instructions into extended-EVEX "map 4", and those keep legacy displacement semantics: disp8 is a plain byte offset. We were applying the vector scaling to them anyway, so `0x50(%rdi)` encoded as disp8 = 0x50 / 16 = 0x05 and the CPU would read it as `0x5(%rdi)`. Found by the XED fuzz oracle; capstone cannot decode map 4 at all, and the memory-operand form is not reachable from lowering yet, so nothing else would have caught it. Key the exemption off `ApxClass::LegacyGpr` rather than the map number. Promoted vector instructions and APX-extended AVX-512 instructions are still vector encodings and do use compressed displacements. --- .../assembler-x64/meta/src/generate/format.rs | 71 +++++++++++-------- cranelift/assembler-x64/src/fuzz.rs | 26 +++++++ 2 files changed, 68 insertions(+), 29 deletions(-) diff --git a/cranelift/assembler-x64/meta/src/generate/format.rs b/cranelift/assembler-x64/meta/src/generate/format.rs index fa4775bb00d9..0e0e88a6efd9 100644 --- a/cranelift/assembler-x64/meta/src/generate/format.rs +++ b/cranelift/assembler-x64/meta/src/generate/format.rs @@ -310,35 +310,48 @@ impl dsl::Format { // Figure out, according to table 2-34 and 2-35 in the Intel manual, // what the scaling factor is for 8-bit displacements to pass through to // encoding. - let evex_scaling = Some(match evex.tuple_type { - dsl::TupleType::Full => { - assert!(!bcast); - length_bytes - } - dsl::TupleType::Half => { - assert!(!bcast); - length_bytes / 2 - } - dsl::TupleType::FullMem => length_bytes, - // FIXME: according to table 2-35 this needs to take into account - // "InputSize" which isn't accounted for in our `Evex` structure at - // this time. - dsl::TupleType::Tuple1Scalar => unimplemented!(), - dsl::TupleType::Tuple1Fixed => unimplemented!(), - dsl::TupleType::Tuple2 => unimplemented!(), - dsl::TupleType::Tuple4 => unimplemented!(), - dsl::TupleType::Tuple8 => 32, - dsl::TupleType::HalfMem => length_bytes / 2, - dsl::TupleType::QuarterMem => length_bytes / 4, - dsl::TupleType::EigthMem => length_bytes / 8, - dsl::TupleType::Mem128 => 16, - dsl::TupleType::Movddup => match evex.length { - dsl::Length::LZ | dsl::Length::LIG => unimplemented!(), - dsl::Length::L128 => 8, - dsl::Length::L256 => 32, - dsl::Length::L512 => 64, - }, - }); + // + // The compressed-displacement scheme of section 2.7.5 only applies to + // the vector EVEX encodings. APX promotes legacy general-purpose- + // register instructions into extended-EVEX "map 4", and those keep + // legacy displacement semantics: `disp8` is a plain byte offset rather + // than a multiple of a tuple-derived scaling factor `N`. Scaling them + // would emit an offset wrong by a factor of `N`. Note this is specific + // to the legacy-GPR class; promoted vector instructions and APX-extended + // AVX-512 instructions still use compressed displacements. + let evex_scaling = if matches!(evex.apx, Some(dsl::ApxClass::LegacyGpr)) { + None + } else { + Some(match evex.tuple_type { + dsl::TupleType::Full => { + assert!(!bcast); + length_bytes + } + dsl::TupleType::Half => { + assert!(!bcast); + length_bytes / 2 + } + dsl::TupleType::FullMem => length_bytes, + // FIXME: according to table 2-35 this needs to take into account + // "InputSize" which isn't accounted for in our `Evex` structure at + // this time. + dsl::TupleType::Tuple1Scalar => unimplemented!(), + dsl::TupleType::Tuple1Fixed => unimplemented!(), + dsl::TupleType::Tuple2 => unimplemented!(), + dsl::TupleType::Tuple4 => unimplemented!(), + dsl::TupleType::Tuple8 => 32, + dsl::TupleType::HalfMem => length_bytes / 2, + dsl::TupleType::QuarterMem => length_bytes / 4, + dsl::TupleType::EigthMem => length_bytes / 8, + dsl::TupleType::Mem128 => 16, + dsl::TupleType::Movddup => match evex.length { + dsl::Length::LZ | dsl::Length::LIG => unimplemented!(), + dsl::Length::L128 => 8, + dsl::Length::L256 => 32, + dsl::Length::L512 => 64, + }, + }) + }; self.generate_vex_or_evex_prefix(f, "EvexPrefix", &bits, is4, evex_scaling, two_op, three_op, || { evex.unwrap_digit() diff --git a/cranelift/assembler-x64/src/fuzz.rs b/cranelift/assembler-x64/src/fuzz.rs index 9eca241194ed..c88e7959f616 100644 --- a/cranelift/assembler-x64/src/fuzz.rs +++ b/cranelift/assembler-x64/src/fuzz.rs @@ -629,4 +629,30 @@ mod test { let assembled = assemble(&inst.into()); assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1801CA"); } + + /// Companion to [`apx_addq_rvm_ndd_encoding`] covering a memory operand. + /// + /// APX map-4 instructions are legacy instructions promoted into EVEX, so + /// their `disp8` keeps legacy semantics: a plain byte offset. The + /// compressed-displacement scheme that divides `disp8` by a tuple-derived + /// factor `N` applies only to the vector EVEX encodings. Encoding + /// `0x50(%rdi)` must therefore emit `0x50` and not `0x50 / 16 = 0x05`, + /// which would silently address the wrong memory. + #[test] + fn apx_addq_rvm_ndd_disp8_is_unscaled() { + use crate::inst::addq_rvm; + use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem}; + + let mem: GprMem = GprMem::Mem(Amode::ImmReg { + base: FuzzReg::new(7), + simm32: AmodeOffsetPlusKnownOffset { + simm32: AmodeOffset::new(0x50), + offset: None, + }, + trap: None, + }); + let inst = addq_rvm::::new(FuzzReg::new(7), FuzzReg::new(7), mem); + let assembled = assemble(&inst.into()); + assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418017F50"); + } } From 9f5a71fd4d82570b6c8c11755ffc70132a45879e Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Fri, 21 Aug 2026 17:28:06 -0700 Subject: [PATCH 8/9] Satisfy rustfmt after widening the prefix helper signature Adding the `two_op`/`three_op` parameters to `generate_vex_or_evex_prefix` pushed both call sites past the width limit, so rustfmt wants them broken across lines. No functional change. --- .../assembler-x64/meta/src/generate/format.rs | 26 ++++++++++++++----- cranelift/assembler-x64/src/fuzz.rs | 3 +-- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/cranelift/assembler-x64/meta/src/generate/format.rs b/cranelift/assembler-x64/meta/src/generate/format.rs index 0e0e88a6efd9..6cb6c1cd5b71 100644 --- a/cranelift/assembler-x64/meta/src/generate/format.rs +++ b/cranelift/assembler-x64/meta/src/generate/format.rs @@ -260,9 +260,16 @@ impl dsl::Format { fmtln!(f, "let w = {};", vex.w.as_bool()); let bits = "len, pp, mmmmm, w"; - self.generate_vex_or_evex_prefix(f, "VexPrefix", &bits, vex.is4, None, "two_op", "three_op", || { - vex.unwrap_digit() - }) + self.generate_vex_or_evex_prefix( + f, + "VexPrefix", + &bits, + vex.is4, + None, + "two_op", + "three_op", + || vex.unwrap_digit(), + ) } fn generate_evex_prefix(&self, f: &mut Formatter, evex: &dsl::Evex) -> ModRmStyle { @@ -353,9 +360,16 @@ impl dsl::Format { }) }; - self.generate_vex_or_evex_prefix(f, "EvexPrefix", &bits, is4, evex_scaling, two_op, three_op, || { - evex.unwrap_digit() - }) + self.generate_vex_or_evex_prefix( + f, + "EvexPrefix", + &bits, + is4, + evex_scaling, + two_op, + three_op, + || evex.unwrap_digit(), + ) } /// Helper function to generate either a vex or evex prefix, mostly handling diff --git a/cranelift/assembler-x64/src/fuzz.rs b/cranelift/assembler-x64/src/fuzz.rs index c88e7959f616..99166feca756 100644 --- a/cranelift/assembler-x64/src/fuzz.rs +++ b/cranelift/assembler-x64/src/fuzz.rs @@ -624,8 +624,7 @@ mod test { // Format is `RVM` = [ModRM.reg source, vvvv destination, ModRM.rm // source], so the constructor arguments are (reg source = %rcx, // destination = %rax, r/m source = %rdx). - let inst = - addq_rvm::::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2)); + let inst = addq_rvm::::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2)); let assembled = assemble(&inst.into()); assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1801CA"); } From d423756c30e3285e1324ba8960379f0759311182 Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Fri, 21 Aug 2026 17:28:33 -0700 Subject: [PATCH 9/9] prtest:full