From fd3aa3801a6bfdefd5b4c9f22273ccce7697494e Mon Sep 17 00:00:00 2001 From: sayantn Date: Sun, 31 May 2026 01:21:36 +0530 Subject: [PATCH 1/2] Correct some wrong uses of LLVM intrinsics --- .../core_arch/src/aarch64/neon/generated.rs | 2 +- crates/core_arch/src/aarch64/prefetch.rs | 2 +- crates/core_arch/src/aarch64/sve/generated.rs | 386 ++++++++++-------- crates/core_arch/src/nvptx/mod.rs | 6 +- crates/core_arch/src/wasm32/memory.rs | 6 +- crates/core_arch/src/x86/sse.rs | 2 +- .../spec/neon/aarch64.spec.yml | 4 +- .../stdarch-gen-arm/spec/sve/aarch64.spec.yml | 59 ++- 8 files changed, 264 insertions(+), 203 deletions(-) diff --git a/crates/core_arch/src/aarch64/neon/generated.rs b/crates/core_arch/src/aarch64/neon/generated.rs index 8d701d9b88..1b5b17e538 100644 --- a/crates/core_arch/src/aarch64/neon/generated.rs +++ b/crates/core_arch/src/aarch64/neon/generated.rs @@ -11903,7 +11903,7 @@ pub unsafe fn vluti4q_lane_s8(a: int8x16_t, b: uint8x8_t) -> in unsafe extern "unadjusted" { #[cfg_attr( any(target_arch = "aarch64", target_arch = "arm64ec"), - link_name = "llvm.aarch64.neon.vluti4q.lane.v8i8" + link_name = "llvm.aarch64.neon.vluti4q.lane.v16i8" )] fn _vluti4q_lane_s8(a: int8x16_t, b: uint8x8_t, n: i32) -> int8x16_t; } diff --git a/crates/core_arch/src/aarch64/prefetch.rs b/crates/core_arch/src/aarch64/prefetch.rs index 4dcbc9549f..88e5c0987f 100644 --- a/crates/core_arch/src/aarch64/prefetch.rs +++ b/crates/core_arch/src/aarch64/prefetch.rs @@ -2,7 +2,7 @@ use stdarch_test::assert_instr; unsafe extern "unadjusted" { - #[link_name = "llvm.prefetch"] + #[link_name = "llvm.prefetch.p0"] fn prefetch(p: *const i8, rw: i32, loc: i32, ty: i32); } diff --git a/crates/core_arch/src/aarch64/sve/generated.rs b/crates/core_arch/src/aarch64/sve/generated.rs index 6c6a2476a4..6c5d5692a1 100644 --- a/crates/core_arch/src/aarch64/sve/generated.rs +++ b/crates/core_arch/src/aarch64/sve/generated.rs @@ -1843,7 +1843,7 @@ pub fn svadrd_u64base_u64index(bases: svuint64_t, indices: svuint64_t) -> svuint #[cfg_attr(test, assert_instr(and))] pub fn svand_b_z(pg: svbool_t, op1: svbool_t, op2: svbool_t) -> svbool_t { unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.and.z.nvx16i1")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.and.z.nxv16i1")] fn _svand_b_z(pg: svbool_t, op1: svbool_t, op2: svbool_t) -> svbool_t; } unsafe { _svand_b_z(pg, op1, op2) } @@ -2935,7 +2935,7 @@ pub fn svasrd_n_s64_z(pg: svbool_t, op1: svint64_t) -> svint64_ #[cfg_attr(test, assert_instr(bic))] pub fn svbic_b_z(pg: svbool_t, op1: svbool_t, op2: svbool_t) -> svbool_t { unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.bic.z.nvx16i1")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.bic.z.nxv16i1")] fn _svbic_b_z(pg: svbool_t, op1: svbool_t, op2: svbool_t) -> svbool_t; } unsafe { _svbic_b_z(pg, op1, op2) } @@ -4559,7 +4559,7 @@ pub fn svcmla_lane_f32( unsafe extern "unadjusted" { #[cfg_attr( target_arch = "aarch64", - link_name = "llvm.aarch64.sve.fcmla.lane.x.nxv4f32" + link_name = "llvm.aarch64.sve.fcmla.lane.nxv4f32" )] fn _svcmla_lane_f32( op1: svfloat32_t, @@ -7657,7 +7657,10 @@ pub fn svcvt_f64_f32_z(pg: svbool_t, op: svfloat32_t) -> svfloat64_t { #[cfg_attr(test, assert_instr(scvtf))] pub fn svcvt_f32_s32_m(inactive: svfloat32_t, pg: svbool_t, op: svint32_t) -> svfloat32_t { unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.scvtf.f32i32")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.sve.scvtf.nxv4f32.nxv4i32" + )] fn _svcvt_f32_s32_m(inactive: svfloat32_t, pg: svbool4_t, op: svint32_t) -> svfloat32_t; } unsafe { _svcvt_f32_s32_m(inactive, pg.sve_into(), op) } @@ -7681,66 +7684,137 @@ pub fn svcvt_f32_s32_z(pg: svbool_t, op: svint32_t) -> svfloat32_t { svcvt_f32_s32_m(svdup_n_f32(0.0), pg, op) } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f32[_s64]_m)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f32[_u32]_m)"] +#[inline] +#[target_feature(enable = "sve")] +#[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] +#[cfg_attr(test, assert_instr(ucvtf))] +pub fn svcvt_f32_u32_m(inactive: svfloat32_t, pg: svbool_t, op: svuint32_t) -> svfloat32_t { + unsafe extern "unadjusted" { + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.sve.ucvtf.nxv4f32.nxv4i32" + )] + fn _svcvt_f32_u32_m(inactive: svfloat32_t, pg: svbool4_t, op: svint32_t) -> svfloat32_t; + } + unsafe { _svcvt_f32_u32_m(inactive, pg.sve_into(), op.as_signed()) } +} +#[doc = "Floating-point convert"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f32[_u32]_x)"] +#[inline] +#[target_feature(enable = "sve")] +#[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] +#[cfg_attr(test, assert_instr(ucvtf))] +pub fn svcvt_f32_u32_x(pg: svbool_t, op: svuint32_t) -> svfloat32_t { + unsafe { svcvt_f32_u32_m(transmute_unchecked(op), pg, op) } +} +#[doc = "Floating-point convert"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f32[_u32]_z)"] +#[inline] +#[target_feature(enable = "sve")] +#[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] +#[cfg_attr(test, assert_instr(ucvtf))] +pub fn svcvt_f32_u32_z(pg: svbool_t, op: svuint32_t) -> svfloat32_t { + svcvt_f32_u32_m(svdup_n_f32(0.0), pg, op) +} +#[doc = "Floating-point convert"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f64[_s64]_m)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] #[cfg_attr(test, assert_instr(scvtf))] -pub fn svcvt_f32_s64_m(inactive: svfloat32_t, pg: svbool_t, op: svint64_t) -> svfloat32_t { +pub fn svcvt_f64_s64_m(inactive: svfloat64_t, pg: svbool_t, op: svint64_t) -> svfloat64_t { unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.scvtf.f32i64")] - fn _svcvt_f32_s64_m(inactive: svfloat32_t, pg: svbool2_t, op: svint64_t) -> svfloat32_t; + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.sve.scvtf.nxv2f64.nxv2i64" + )] + fn _svcvt_f64_s64_m(inactive: svfloat64_t, pg: svbool2_t, op: svint64_t) -> svfloat64_t; } - unsafe { _svcvt_f32_s64_m(inactive, pg.sve_into(), op) } + unsafe { _svcvt_f64_s64_m(inactive, pg.sve_into(), op) } } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f32[_s64]_x)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f64[_s64]_x)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] #[cfg_attr(test, assert_instr(scvtf))] -pub fn svcvt_f32_s64_x(pg: svbool_t, op: svint64_t) -> svfloat32_t { - unsafe { svcvt_f32_s64_m(transmute_unchecked(op), pg, op) } +pub fn svcvt_f64_s64_x(pg: svbool_t, op: svint64_t) -> svfloat64_t { + unsafe { svcvt_f64_s64_m(transmute_unchecked(op), pg, op) } } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f32[_s64]_z)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f64[_s64]_z)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] #[cfg_attr(test, assert_instr(scvtf))] -pub fn svcvt_f32_s64_z(pg: svbool_t, op: svint64_t) -> svfloat32_t { - svcvt_f32_s64_m(svdup_n_f32(0.0), pg, op) +pub fn svcvt_f64_s64_z(pg: svbool_t, op: svint64_t) -> svfloat64_t { + svcvt_f64_s64_m(svdup_n_f64(0.0), pg, op) } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f32[_u32]_m)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f64[_u64]_m)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] #[cfg_attr(test, assert_instr(ucvtf))] -pub fn svcvt_f32_u32_m(inactive: svfloat32_t, pg: svbool_t, op: svuint32_t) -> svfloat32_t { +pub fn svcvt_f64_u64_m(inactive: svfloat64_t, pg: svbool_t, op: svuint64_t) -> svfloat64_t { unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.ucvtf.f32i32")] - fn _svcvt_f32_u32_m(inactive: svfloat32_t, pg: svbool4_t, op: svint32_t) -> svfloat32_t; + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.sve.ucvtf.nxv2f64.nxv2i64" + )] + fn _svcvt_f64_u64_m(inactive: svfloat64_t, pg: svbool2_t, op: svint64_t) -> svfloat64_t; } - unsafe { _svcvt_f32_u32_m(inactive, pg.sve_into(), op.as_signed()) } + unsafe { _svcvt_f64_u64_m(inactive, pg.sve_into(), op.as_signed()) } } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f32[_u32]_x)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f64[_u64]_x)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] #[cfg_attr(test, assert_instr(ucvtf))] -pub fn svcvt_f32_u32_x(pg: svbool_t, op: svuint32_t) -> svfloat32_t { - unsafe { svcvt_f32_u32_m(transmute_unchecked(op), pg, op) } +pub fn svcvt_f64_u64_x(pg: svbool_t, op: svuint64_t) -> svfloat64_t { + unsafe { svcvt_f64_u64_m(transmute_unchecked(op), pg, op) } } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f32[_u32]_z)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f64[_u64]_z)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] #[cfg_attr(test, assert_instr(ucvtf))] -pub fn svcvt_f32_u32_z(pg: svbool_t, op: svuint32_t) -> svfloat32_t { - svcvt_f32_u32_m(svdup_n_f32(0.0), pg, op) +pub fn svcvt_f64_u64_z(pg: svbool_t, op: svuint64_t) -> svfloat64_t { + svcvt_f64_u64_m(svdup_n_f64(0.0), pg, op) +} +#[doc = "Floating-point convert"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f32[_s64]_m)"] +#[inline] +#[target_feature(enable = "sve")] +#[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] +#[cfg_attr(test, assert_instr(scvtf))] +pub fn svcvt_f32_s64_m(inactive: svfloat32_t, pg: svbool_t, op: svint64_t) -> svfloat32_t { + unsafe extern "unadjusted" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.scvtf.f32i64")] + fn _svcvt_f32_s64_m(inactive: svfloat32_t, pg: svbool2_t, op: svint64_t) -> svfloat32_t; + } + unsafe { _svcvt_f32_s64_m(inactive, pg.sve_into(), op) } +} +#[doc = "Floating-point convert"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f32[_s64]_x)"] +#[inline] +#[target_feature(enable = "sve")] +#[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] +#[cfg_attr(test, assert_instr(scvtf))] +pub fn svcvt_f32_s64_x(pg: svbool_t, op: svint64_t) -> svfloat32_t { + unsafe { svcvt_f32_s64_m(transmute_unchecked(op), pg, op) } +} +#[doc = "Floating-point convert"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f32[_s64]_z)"] +#[inline] +#[target_feature(enable = "sve")] +#[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] +#[cfg_attr(test, assert_instr(scvtf))] +pub fn svcvt_f32_s64_z(pg: svbool_t, op: svint64_t) -> svfloat32_t { + svcvt_f32_s64_m(svdup_n_f32(0.0), pg, op) } #[doc = "Floating-point convert"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f32[_u64]_m)"] @@ -7805,37 +7879,6 @@ pub fn svcvt_f64_s32_z(pg: svbool_t, op: svint32_t) -> svfloat64_t { svcvt_f64_s32_m(svdup_n_f64(0.0), pg, op) } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f64[_s64]_m)"] -#[inline] -#[target_feature(enable = "sve")] -#[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(scvtf))] -pub fn svcvt_f64_s64_m(inactive: svfloat64_t, pg: svbool_t, op: svint64_t) -> svfloat64_t { - unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.scvtf.f64i64")] - fn _svcvt_f64_s64_m(inactive: svfloat64_t, pg: svbool2_t, op: svint64_t) -> svfloat64_t; - } - unsafe { _svcvt_f64_s64_m(inactive, pg.sve_into(), op) } -} -#[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f64[_s64]_x)"] -#[inline] -#[target_feature(enable = "sve")] -#[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(scvtf))] -pub fn svcvt_f64_s64_x(pg: svbool_t, op: svint64_t) -> svfloat64_t { - unsafe { svcvt_f64_s64_m(transmute_unchecked(op), pg, op) } -} -#[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f64[_s64]_z)"] -#[inline] -#[target_feature(enable = "sve")] -#[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(scvtf))] -pub fn svcvt_f64_s64_z(pg: svbool_t, op: svint64_t) -> svfloat64_t { - svcvt_f64_s64_m(svdup_n_f64(0.0), pg, op) -} -#[doc = "Floating-point convert"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f64[_u32]_m)"] #[inline] #[target_feature(enable = "sve")] @@ -7867,190 +7910,202 @@ pub fn svcvt_f64_u32_z(pg: svbool_t, op: svuint32_t) -> svfloat64_t { svcvt_f64_u32_m(svdup_n_f64(0.0), pg, op) } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f64[_u64]_m)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s32[_f32]_m)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(ucvtf))] -pub fn svcvt_f64_u64_m(inactive: svfloat64_t, pg: svbool_t, op: svuint64_t) -> svfloat64_t { +#[cfg_attr(test, assert_instr(fcvtzs))] +pub fn svcvt_s32_f32_m(inactive: svint32_t, pg: svbool_t, op: svfloat32_t) -> svint32_t { unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.ucvtf.f64i64")] - fn _svcvt_f64_u64_m(inactive: svfloat64_t, pg: svbool2_t, op: svint64_t) -> svfloat64_t; + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.sve.fcvtzs.nxv4i32.nxv4f32" + )] + fn _svcvt_s32_f32_m(inactive: svint32_t, pg: svbool4_t, op: svfloat32_t) -> svint32_t; } - unsafe { _svcvt_f64_u64_m(inactive, pg.sve_into(), op.as_signed()) } + unsafe { _svcvt_s32_f32_m(inactive, pg.sve_into(), op) } } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f64[_u64]_x)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s32[_f32]_x)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(ucvtf))] -pub fn svcvt_f64_u64_x(pg: svbool_t, op: svuint64_t) -> svfloat64_t { - unsafe { svcvt_f64_u64_m(transmute_unchecked(op), pg, op) } +#[cfg_attr(test, assert_instr(fcvtzs))] +pub fn svcvt_s32_f32_x(pg: svbool_t, op: svfloat32_t) -> svint32_t { + unsafe { svcvt_s32_f32_m(transmute_unchecked(op), pg, op) } } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_f64[_u64]_z)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s32[_f32]_z)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(ucvtf))] -pub fn svcvt_f64_u64_z(pg: svbool_t, op: svuint64_t) -> svfloat64_t { - svcvt_f64_u64_m(svdup_n_f64(0.0), pg, op) +#[cfg_attr(test, assert_instr(fcvtzs))] +pub fn svcvt_s32_f32_z(pg: svbool_t, op: svfloat32_t) -> svint32_t { + svcvt_s32_f32_m(svdup_n_s32(0), pg, op) } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s32[_f32]_m)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s64[_f64]_m)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] #[cfg_attr(test, assert_instr(fcvtzs))] -pub fn svcvt_s32_f32_m(inactive: svint32_t, pg: svbool_t, op: svfloat32_t) -> svint32_t { +pub fn svcvt_s64_f64_m(inactive: svint64_t, pg: svbool_t, op: svfloat64_t) -> svint64_t { unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.fcvtzs.i32f32")] - fn _svcvt_s32_f32_m(inactive: svint32_t, pg: svbool4_t, op: svfloat32_t) -> svint32_t; + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.sve.fcvtzs.nxv2i64.nxv2f64" + )] + fn _svcvt_s64_f64_m(inactive: svint64_t, pg: svbool2_t, op: svfloat64_t) -> svint64_t; } - unsafe { _svcvt_s32_f32_m(inactive, pg.sve_into(), op) } + unsafe { _svcvt_s64_f64_m(inactive, pg.sve_into(), op) } } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s32[_f32]_x)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s64[_f64]_x)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] #[cfg_attr(test, assert_instr(fcvtzs))] -pub fn svcvt_s32_f32_x(pg: svbool_t, op: svfloat32_t) -> svint32_t { - unsafe { svcvt_s32_f32_m(transmute_unchecked(op), pg, op) } +pub fn svcvt_s64_f64_x(pg: svbool_t, op: svfloat64_t) -> svint64_t { + unsafe { svcvt_s64_f64_m(transmute_unchecked(op), pg, op) } } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s32[_f32]_z)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s64[_f64]_z)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] #[cfg_attr(test, assert_instr(fcvtzs))] -pub fn svcvt_s32_f32_z(pg: svbool_t, op: svfloat32_t) -> svint32_t { - svcvt_s32_f32_m(svdup_n_s32(0), pg, op) +pub fn svcvt_s64_f64_z(pg: svbool_t, op: svfloat64_t) -> svint64_t { + svcvt_s64_f64_m(svdup_n_s64(0), pg, op) } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s32[_f64]_m)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_u32[_f32]_m)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(fcvtzs))] -pub fn svcvt_s32_f64_m(inactive: svint32_t, pg: svbool_t, op: svfloat64_t) -> svint32_t { +#[cfg_attr(test, assert_instr(fcvtzu))] +pub fn svcvt_u32_f32_m(inactive: svuint32_t, pg: svbool_t, op: svfloat32_t) -> svuint32_t { unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.fcvtzs.i32f64")] - fn _svcvt_s32_f64_m(inactive: svint32_t, pg: svbool2_t, op: svfloat64_t) -> svint32_t; + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.sve.fcvtzu.nxv4i32.nxv4f32" + )] + fn _svcvt_u32_f32_m(inactive: svint32_t, pg: svbool4_t, op: svfloat32_t) -> svint32_t; } - unsafe { _svcvt_s32_f64_m(inactive, pg.sve_into(), op) } + unsafe { _svcvt_u32_f32_m(inactive.as_signed(), pg.sve_into(), op).as_unsigned() } } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s32[_f64]_x)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_u32[_f32]_x)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(fcvtzs))] -pub fn svcvt_s32_f64_x(pg: svbool_t, op: svfloat64_t) -> svint32_t { - unsafe { svcvt_s32_f64_m(transmute_unchecked(op), pg, op) } +#[cfg_attr(test, assert_instr(fcvtzu))] +pub fn svcvt_u32_f32_x(pg: svbool_t, op: svfloat32_t) -> svuint32_t { + unsafe { svcvt_u32_f32_m(transmute_unchecked(op), pg, op) } } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s32[_f64]_z)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_u32[_f32]_z)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(fcvtzs))] -pub fn svcvt_s32_f64_z(pg: svbool_t, op: svfloat64_t) -> svint32_t { - svcvt_s32_f64_m(svdup_n_s32(0), pg, op) +#[cfg_attr(test, assert_instr(fcvtzu))] +pub fn svcvt_u32_f32_z(pg: svbool_t, op: svfloat32_t) -> svuint32_t { + svcvt_u32_f32_m(svdup_n_u32(0), pg, op) } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s64[_f32]_m)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_u64[_f64]_m)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(fcvtzs))] -pub fn svcvt_s64_f32_m(inactive: svint64_t, pg: svbool_t, op: svfloat32_t) -> svint64_t { +#[cfg_attr(test, assert_instr(fcvtzu))] +pub fn svcvt_u64_f64_m(inactive: svuint64_t, pg: svbool_t, op: svfloat64_t) -> svuint64_t { unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.fcvtzs.i64f32")] - fn _svcvt_s64_f32_m(inactive: svint64_t, pg: svbool2_t, op: svfloat32_t) -> svint64_t; + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.sve.fcvtzu.nxv2i64.nxv2f64" + )] + fn _svcvt_u64_f64_m(inactive: svint64_t, pg: svbool2_t, op: svfloat64_t) -> svint64_t; } - unsafe { _svcvt_s64_f32_m(inactive, pg.sve_into(), op) } + unsafe { _svcvt_u64_f64_m(inactive.as_signed(), pg.sve_into(), op).as_unsigned() } } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s64[_f32]_x)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_u64[_f64]_x)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(fcvtzs))] -pub fn svcvt_s64_f32_x(pg: svbool_t, op: svfloat32_t) -> svint64_t { - unsafe { svcvt_s64_f32_m(transmute_unchecked(op), pg, op) } +#[cfg_attr(test, assert_instr(fcvtzu))] +pub fn svcvt_u64_f64_x(pg: svbool_t, op: svfloat64_t) -> svuint64_t { + unsafe { svcvt_u64_f64_m(transmute_unchecked(op), pg, op) } } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s64[_f32]_z)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_u64[_f64]_z)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(fcvtzs))] -pub fn svcvt_s64_f32_z(pg: svbool_t, op: svfloat32_t) -> svint64_t { - svcvt_s64_f32_m(svdup_n_s64(0), pg, op) +#[cfg_attr(test, assert_instr(fcvtzu))] +pub fn svcvt_u64_f64_z(pg: svbool_t, op: svfloat64_t) -> svuint64_t { + svcvt_u64_f64_m(svdup_n_u64(0), pg, op) } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s64[_f64]_m)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s32[_f64]_m)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] #[cfg_attr(test, assert_instr(fcvtzs))] -pub fn svcvt_s64_f64_m(inactive: svint64_t, pg: svbool_t, op: svfloat64_t) -> svint64_t { +pub fn svcvt_s32_f64_m(inactive: svint32_t, pg: svbool_t, op: svfloat64_t) -> svint32_t { unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.fcvtzs.i64f64")] - fn _svcvt_s64_f64_m(inactive: svint64_t, pg: svbool2_t, op: svfloat64_t) -> svint64_t; + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.fcvtzs.i32f64")] + fn _svcvt_s32_f64_m(inactive: svint32_t, pg: svbool2_t, op: svfloat64_t) -> svint32_t; } - unsafe { _svcvt_s64_f64_m(inactive, pg.sve_into(), op) } + unsafe { _svcvt_s32_f64_m(inactive, pg.sve_into(), op) } } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s64[_f64]_x)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s32[_f64]_x)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] #[cfg_attr(test, assert_instr(fcvtzs))] -pub fn svcvt_s64_f64_x(pg: svbool_t, op: svfloat64_t) -> svint64_t { - unsafe { svcvt_s64_f64_m(transmute_unchecked(op), pg, op) } +pub fn svcvt_s32_f64_x(pg: svbool_t, op: svfloat64_t) -> svint32_t { + unsafe { svcvt_s32_f64_m(transmute_unchecked(op), pg, op) } } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s64[_f64]_z)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s32[_f64]_z)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] #[cfg_attr(test, assert_instr(fcvtzs))] -pub fn svcvt_s64_f64_z(pg: svbool_t, op: svfloat64_t) -> svint64_t { - svcvt_s64_f64_m(svdup_n_s64(0), pg, op) +pub fn svcvt_s32_f64_z(pg: svbool_t, op: svfloat64_t) -> svint32_t { + svcvt_s32_f64_m(svdup_n_s32(0), pg, op) } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_u32[_f32]_m)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s64[_f32]_m)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(fcvtzu))] -pub fn svcvt_u32_f32_m(inactive: svuint32_t, pg: svbool_t, op: svfloat32_t) -> svuint32_t { +#[cfg_attr(test, assert_instr(fcvtzs))] +pub fn svcvt_s64_f32_m(inactive: svint64_t, pg: svbool_t, op: svfloat32_t) -> svint64_t { unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.fcvtzu.i32f32")] - fn _svcvt_u32_f32_m(inactive: svint32_t, pg: svbool4_t, op: svfloat32_t) -> svint32_t; + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.fcvtzs.i64f32")] + fn _svcvt_s64_f32_m(inactive: svint64_t, pg: svbool2_t, op: svfloat32_t) -> svint64_t; } - unsafe { _svcvt_u32_f32_m(inactive.as_signed(), pg.sve_into(), op).as_unsigned() } + unsafe { _svcvt_s64_f32_m(inactive, pg.sve_into(), op) } } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_u32[_f32]_x)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s64[_f32]_x)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(fcvtzu))] -pub fn svcvt_u32_f32_x(pg: svbool_t, op: svfloat32_t) -> svuint32_t { - unsafe { svcvt_u32_f32_m(transmute_unchecked(op), pg, op) } +#[cfg_attr(test, assert_instr(fcvtzs))] +pub fn svcvt_s64_f32_x(pg: svbool_t, op: svfloat32_t) -> svint64_t { + unsafe { svcvt_s64_f32_m(transmute_unchecked(op), pg, op) } } #[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_u32[_f32]_z)"] +#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_s64[_f32]_z)"] #[inline] #[target_feature(enable = "sve")] #[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(fcvtzu))] -pub fn svcvt_u32_f32_z(pg: svbool_t, op: svfloat32_t) -> svuint32_t { - svcvt_u32_f32_m(svdup_n_u32(0), pg, op) +#[cfg_attr(test, assert_instr(fcvtzs))] +pub fn svcvt_s64_f32_z(pg: svbool_t, op: svfloat32_t) -> svint64_t { + svcvt_s64_f32_m(svdup_n_s64(0), pg, op) } #[doc = "Floating-point convert"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_u32[_f64]_m)"] @@ -8114,37 +8169,6 @@ pub fn svcvt_u64_f32_x(pg: svbool_t, op: svfloat32_t) -> svuint64_t { pub fn svcvt_u64_f32_z(pg: svbool_t, op: svfloat32_t) -> svuint64_t { svcvt_u64_f32_m(svdup_n_u64(0), pg, op) } -#[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_u64[_f64]_m)"] -#[inline] -#[target_feature(enable = "sve")] -#[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(fcvtzu))] -pub fn svcvt_u64_f64_m(inactive: svuint64_t, pg: svbool_t, op: svfloat64_t) -> svuint64_t { - unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.fcvtzu.i64f64")] - fn _svcvt_u64_f64_m(inactive: svint64_t, pg: svbool2_t, op: svfloat64_t) -> svint64_t; - } - unsafe { _svcvt_u64_f64_m(inactive.as_signed(), pg.sve_into(), op).as_unsigned() } -} -#[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_u64[_f64]_x)"] -#[inline] -#[target_feature(enable = "sve")] -#[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(fcvtzu))] -pub fn svcvt_u64_f64_x(pg: svbool_t, op: svfloat64_t) -> svuint64_t { - unsafe { svcvt_u64_f64_m(transmute_unchecked(op), pg, op) } -} -#[doc = "Floating-point convert"] -#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svcvt_u64[_f64]_z)"] -#[inline] -#[target_feature(enable = "sve")] -#[unstable(feature = "stdarch_aarch64_sve", issue = "145052")] -#[cfg_attr(test, assert_instr(fcvtzu))] -pub fn svcvt_u64_f64_z(pg: svbool_t, op: svfloat64_t) -> svuint64_t { - svcvt_u64_f64_m(svdup_n_u64(0), pg, op) -} #[doc = "Divide"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svdiv[_f32]_m)"] #[inline] @@ -10040,7 +10064,7 @@ pub fn svdupq_n_u8( #[cfg_attr(test, assert_instr(eor))] pub fn sveor_b_z(pg: svbool_t, op1: svbool_t, op2: svbool_t) -> svbool_t { unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.eor.z.nvx16i1")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.eor.z.nxv16i1")] fn _sveor_b_z(pg: svbool_t, op1: svbool_t, op2: svbool_t) -> svbool_t; } unsafe { _sveor_b_z(pg, op1, op2) } @@ -10591,7 +10615,7 @@ pub fn svexpa_f32(op: svuint32_t) -> svfloat32_t { unsafe extern "unadjusted" { #[cfg_attr( target_arch = "aarch64", - link_name = "llvm.aarch64.sve.fexpa.x.nxv4f32 " + link_name = "llvm.aarch64.sve.fexpa.x.nxv4f32" )] fn _svexpa_f32(op: svint32_t) -> svfloat32_t; } @@ -10607,7 +10631,7 @@ pub fn svexpa_f64(op: svuint64_t) -> svfloat64_t { unsafe extern "unadjusted" { #[cfg_attr( target_arch = "aarch64", - link_name = "llvm.aarch64.sve.fexpa.x.nxv2f64 " + link_name = "llvm.aarch64.sve.fexpa.x.nxv2f64" )] fn _svexpa_f64(op: svint64_t) -> svfloat64_t; } @@ -27371,7 +27395,10 @@ pub fn svmls_lane_f64( #[cfg_attr(test, assert_instr(fmmla))] pub fn svmmla_f32(op1: svfloat32_t, op2: svfloat32_t, op3: svfloat32_t) -> svfloat32_t { unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.fmmla.nxv4f32")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.sve.fmmla.nxv4f32.nxv4f32" + )] fn _svmmla_f32(op1: svfloat32_t, op2: svfloat32_t, op3: svfloat32_t) -> svfloat32_t; } unsafe { _svmmla_f32(op1, op2, op3) } @@ -27384,7 +27411,10 @@ pub fn svmmla_f32(op1: svfloat32_t, op2: svfloat32_t, op3: svfloat32_t) -> svflo #[cfg_attr(test, assert_instr(fmmla))] pub fn svmmla_f64(op1: svfloat64_t, op2: svfloat64_t, op3: svfloat64_t) -> svfloat64_t { unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.fmmla.nxv2f64")] + #[cfg_attr( + target_arch = "aarch64", + link_name = "llvm.aarch64.sve.fmmla.nxv2f64.nxv2f64" + )] fn _svmmla_f64(op1: svfloat64_t, op2: svfloat64_t, op3: svfloat64_t) -> svfloat64_t; } unsafe { _svmmla_f64(op1, op2, op3) } @@ -30260,7 +30290,7 @@ pub fn svnot_u64_z(pg: svbool_t, op: svuint64_t) -> svuint64_t { #[cfg_attr(test, assert_instr(orn))] pub fn svorn_b_z(pg: svbool_t, op1: svbool_t, op2: svbool_t) -> svbool_t { unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.orn.z.nvx16i1")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.orn.z.nxv16i1")] fn _svorn_b_z(pg: svbool_t, op1: svbool_t, op2: svbool_t) -> svbool_t; } unsafe { _svorn_b_z(pg, op1, op2) } @@ -30273,7 +30303,7 @@ pub fn svorn_b_z(pg: svbool_t, op1: svbool_t, op2: svbool_t) -> svbool_t { #[cfg_attr(test, assert_instr(orr))] pub fn svorr_b_z(pg: svbool_t, op1: svbool_t, op2: svbool_t) -> svbool_t { unsafe extern "unadjusted" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.orr.z.nvx16i1")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.orr.z.nxv16i1")] fn _svorr_b_z(pg: svbool_t, op1: svbool_t, op2: svbool_t) -> svbool_t; } unsafe { _svorr_b_z(pg, op1, op2) } @@ -34340,10 +34370,7 @@ pub fn svrecps_f64(op1: svfloat64_t, op2: svfloat64_t) -> svfloat64_t { #[cfg_attr(test, assert_instr(frecpx))] pub fn svrecpx_f32_m(inactive: svfloat32_t, pg: svbool_t, op: svfloat32_t) -> svfloat32_t { unsafe extern "unadjusted" { - #[cfg_attr( - target_arch = "aarch64", - link_name = "llvm.aarch64.sve.frecpx.x.nxv4f32" - )] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.frecpx.nxv4f32")] fn _svrecpx_f32_m(inactive: svfloat32_t, pg: svbool4_t, op: svfloat32_t) -> svfloat32_t; } unsafe { _svrecpx_f32_m(inactive, pg.sve_into(), op) } @@ -34374,10 +34401,7 @@ pub fn svrecpx_f32_z(pg: svbool_t, op: svfloat32_t) -> svfloat32_t { #[cfg_attr(test, assert_instr(frecpx))] pub fn svrecpx_f64_m(inactive: svfloat64_t, pg: svbool_t, op: svfloat64_t) -> svfloat64_t { unsafe extern "unadjusted" { - #[cfg_attr( - target_arch = "aarch64", - link_name = "llvm.aarch64.sve.frecpx.x.nxv2f64" - )] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.frecpx.nxv2f64")] fn _svrecpx_f64_m(inactive: svfloat64_t, pg: svbool2_t, op: svfloat64_t) -> svfloat64_t; } unsafe { _svrecpx_f64_m(inactive, pg.sve_into(), op) } diff --git a/crates/core_arch/src/nvptx/mod.rs b/crates/core_arch/src/nvptx/mod.rs index b63a5d01a7..84d8810320 100644 --- a/crates/core_arch/src/nvptx/mod.rs +++ b/crates/core_arch/src/nvptx/mod.rs @@ -20,8 +20,8 @@ pub use packed::*; #[allow(improper_ctypes)] unsafe extern "C" { - #[link_name = "llvm.nvvm.barrier0"] - fn syncthreads() -> (); + #[link_name = "llvm.nvvm.barrier.cta.sync.aligned.all"] + fn syncthreads(a: u32) -> (); #[link_name = "llvm.nvvm.read.ptx.sreg.ntid.x"] fn block_dim_x() -> u32; #[link_name = "llvm.nvvm.read.ptx.sreg.ntid.y"] @@ -54,7 +54,7 @@ unsafe extern "C" { #[inline] #[unstable(feature = "stdarch_nvptx", issue = "111199")] pub unsafe fn _syncthreads() -> () { - syncthreads() + syncthreads(0) } /// x-th thread-block dimension. diff --git a/crates/core_arch/src/wasm32/memory.rs b/crates/core_arch/src/wasm32/memory.rs index 90e9075e51..9d7b7287ed 100644 --- a/crates/core_arch/src/wasm32/memory.rs +++ b/crates/core_arch/src/wasm32/memory.rs @@ -2,9 +2,11 @@ use stdarch_test::assert_instr; unsafe extern "unadjusted" { - #[link_name = "llvm.wasm.memory.grow"] + #[cfg_attr(target_pointer_width = "32", link_name = "llvm.wasm.memory.grow.i32")] + #[cfg_attr(target_pointer_width = "64", link_name = "llvm.wasm.memory.grow.i64")] fn llvm_memory_grow(mem: u32, pages: usize) -> usize; - #[link_name = "llvm.wasm.memory.size"] + #[cfg_attr(target_pointer_width = "32", link_name = "llvm.wasm.memory.size.i32")] + #[cfg_attr(target_pointer_width = "64", link_name = "llvm.wasm.memory.size.i64")] fn llvm_memory_size(mem: u32) -> usize; } diff --git a/crates/core_arch/src/x86/sse.rs b/crates/core_arch/src/x86/sse.rs index 8673872901..afaaccfbf2 100644 --- a/crates/core_arch/src/x86/sse.rs +++ b/crates/core_arch/src/x86/sse.rs @@ -2040,7 +2040,7 @@ unsafe extern "C" { fn stmxcsr(p: *mut i8); #[link_name = "llvm.x86.sse.ldmxcsr"] fn ldmxcsr(p: *const i8); - #[link_name = "llvm.prefetch"] + #[link_name = "llvm.prefetch.p0"] fn prefetch(p: *const i8, rw: i32, loc: i32, ty: i32); #[link_name = "llvm.x86.sse.cmp.ss"] fn cmpss(a: __m128, b: __m128, imm8: i8) -> __m128; diff --git a/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml b/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml index 03fea5e0a4..e5ce77ed8b 100644 --- a/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml +++ b/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml @@ -13951,7 +13951,7 @@ intrinsics: - 'b: {neon_type[1]}' - 'n: i32' links: - - link: "llvm.aarch64.neon.vluti4q.lane.{neon_type[1]}" + - link: "llvm.aarch64.neon.vluti4q.lane.{neon_type[0]}" arch: aarch64,arm64ec - FnCall: ['_vluti4{neon_type[0].lane_nox}', [a, b, LANE]] @@ -14002,7 +14002,7 @@ intrinsics: - 'b: {neon_type[1]}' - 'n: i32' links: - - link: "llvm.aarch64.neon.vluti4q.laneq.{neon_type[1]}" + - link: "llvm.aarch64.neon.vluti4q.laneq.{neon_type[0]}" arch: aarch64,arm64ec - FnCall: ['_vluti4{neon_type[0].laneq_nox}', [a, b, LANE]] diff --git a/crates/stdarch-gen-arm/spec/sve/aarch64.spec.yml b/crates/stdarch-gen-arm/spec/sve/aarch64.spec.yml index 29dd3a095d..1f65732412 100644 --- a/crates/stdarch-gen-arm/spec/sve/aarch64.spec.yml +++ b/crates/stdarch-gen-arm/spec/sve/aarch64.spec.yml @@ -115,7 +115,7 @@ intrinsics: assert_instr: [[fcmla, "IMM_INDEX = 0, IMM_ROTATION = 90"]] compose: - LLVMLink: - name: fcmla.lane.x.{sve_type} + name: fcmla.lane.{sve_type} arguments: - "op1: {sve_type}" - "op2: {sve_type}" @@ -4179,7 +4179,8 @@ intrinsics: ["inactive: {sve_type[0]}", "pg: {max_predicate}", "op: {sve_type[1]}"] return_type: "{sve_type[0]}" types: - - [[f32, f64], [i32, u32, i64, u64]] + - [f32, [i64, u64]] + - [f64, [i32, u32]] zeroing_method: { drop: inactive } substitutions: convert_from: { match_kind: "{type[1]}", default: s, unsigned: u } @@ -4187,6 +4188,23 @@ intrinsics: compose: - LLVMLink: name: "{convert_from}cvtf.{type[0]}{type[1]}" + + - name: svcvt_{type[0]}[_{type[1]}]{_mxz} + attr: [*sve-unstable] + doc: Floating-point convert + arguments: + ["inactive: {sve_type[0]}", "pg: {max_predicate}", "op: {sve_type[1]}"] + return_type: "{sve_type[0]}" + types: + - [f32, [i32, u32]] + - [f64, [i64, u64]] + zeroing_method: { drop: inactive } + substitutions: + convert_from: { match_kind: "{type[1]}", default: s, unsigned: u } + assert_instr: ["{convert_from}cvtf"] + compose: + - LLVMLink: + name: "{convert_from}cvtf.{sve_type[0]}.{sve_type[1]}" - name: svcvt_{type[0]}[_{type[1]}]{_mxz} attr: [*sve-unstable] @@ -4195,13 +4213,30 @@ intrinsics: ["inactive: {sve_type[0]}", "pg: {max_predicate}", "op: {sve_type[1]}"] return_type: "{sve_type[0]}" types: - - [[i32, u32, i64, u64], [f32, f64]] + - [[i32, u32], f64] + - [[i64, u64], f32] zeroing_method: { drop: inactive } substitutions: convert_to: { match_kind: "{type[0]}", default: s, unsigned: u } assert_instr: ["fcvtz{convert_to}"] compose: - LLVMLink: { name: "fcvtz{convert_to}.{type[0]}{type[1]}" } + + - name: svcvt_{type[0]}[_{type[1]}]{_mxz} + attr: [*sve-unstable] + doc: Floating-point convert + arguments: + ["inactive: {sve_type[0]}", "pg: {max_predicate}", "op: {sve_type[1]}"] + return_type: "{sve_type[0]}" + types: + - [[i32, u32], f32] + - [[i64, u64], f64] + zeroing_method: { drop: inactive } + substitutions: + convert_to: { match_kind: "{type[0]}", default: s, unsigned: u } + assert_instr: ["fcvtz{convert_to}"] + compose: + - LLVMLink: { name: "fcvtz{convert_to}.{sve_type[0]}.{sve_type[1]}" } - name: svcvt_{type[0]}[_{type[1]}]{_mxz} attr: [*sve-unstable] @@ -4356,7 +4391,7 @@ intrinsics: return_type: svbool_t assert_instr: [and] compose: - - LLVMLink: { name: "and.z.nvx16i1" } + - LLVMLink: { name: "and.z.nxv16i1" } - name: svmov[_b]_z attr: [*sve-unstable] @@ -4386,7 +4421,7 @@ intrinsics: return_type: svbool_t assert_instr: [bic] compose: - - LLVMLink: { name: "bic.z.nvx16i1" } + - LLVMLink: { name: "bic.z.nxv16i1" } - name: sveor[{_n}_{type}]{_mxz} attr: [*sve-unstable] @@ -4417,7 +4452,7 @@ intrinsics: return_type: svbool_t assert_instr: [eor] compose: - - LLVMLink: { name: "eor.z.nvx16i1" } + - LLVMLink: { name: "eor.z.nxv16i1" } - name: svnot[_{type}]{_mxz} attr: [*sve-unstable] @@ -4497,7 +4532,7 @@ intrinsics: return_type: svbool_t assert_instr: [orr] compose: - - LLVMLink: { name: "orr.z.nvx16i1" } + - LLVMLink: { name: "orr.z.nxv16i1" } - name: svorn[_b]_z attr: [*sve-unstable] @@ -4506,7 +4541,7 @@ intrinsics: return_type: svbool_t assert_instr: [orn] compose: - - LLVMLink: { name: "orn.z.nvx16i1" } + - LLVMLink: { name: "orn.z.nxv16i1" } - name: svlsl[{_n}_{type[0]}]{_mxz} attr: [*sve-unstable] @@ -4749,7 +4784,7 @@ intrinsics: assert_instr: [frecpx] zeroing_method: { drop: inactive } compose: - - LLVMLink: { name: "frecpx.x.{sve_type}" } + - LLVMLink: { name: "frecpx.{sve_type}" } - name: svrsqrte[_{type}] attr: [*sve-unstable] @@ -5115,7 +5150,7 @@ intrinsics: types: [[f32, u32], [f64, u64]] assert_instr: [fexpa] compose: - - LLVMLink: { name: "fexpa.x.{sve_type[0]} " } + - LLVMLink: { name: "fexpa.x.{sve_type[0]}" } - name: svscale[{_n}_{type[0]}]{_mxz} attr: [*sve-unstable] @@ -5139,7 +5174,7 @@ intrinsics: types: [f32] assert_instr: [fmmla] compose: - - LLVMLink: { name: "fmmla.{sve_type}" } + - LLVMLink: { name: "fmmla.{sve_type}.{sve_type}" } - name: svmmla[_{type}] attr: [*sve-unstable] @@ -5150,7 +5185,7 @@ intrinsics: types: [f64] assert_instr: [fmmla] compose: - - LLVMLink: { name: "fmmla.{sve_type}" } + - LLVMLink: { name: "fmmla.{sve_type}.{sve_type}" } - name: svmmla[_{type[0]}] attr: [*sve-unstable] From 5f21f967726a2744552fa5a33d9b6bbeab4b06d2 Mon Sep 17 00:00:00 2001 From: sayantn Date: Sun, 31 May 2026 12:36:47 +0530 Subject: [PATCH 2/2] Use `unadjusted` ABI everywhere --- crates/core_arch/src/aarch64/sve/mod.rs | 4 ++-- crates/core_arch/src/mips/msa.rs | 2 +- crates/core_arch/src/nvptx/mod.rs | 2 +- crates/core_arch/src/nvptx/packed.rs | 2 +- crates/core_arch/src/powerpc/altivec.rs | 2 +- crates/core_arch/src/powerpc/vsx.rs | 2 +- crates/core_arch/src/powerpc64/vsx.rs | 2 +- crates/core_arch/src/x86/aes.rs | 2 +- crates/core_arch/src/x86/avx.rs | 2 +- crates/core_arch/src/x86/avx2.rs | 2 +- crates/core_arch/src/x86/avx512bf16.rs | 2 +- crates/core_arch/src/x86/avx512bitalg.rs | 2 +- crates/core_arch/src/x86/avx512bw.rs | 2 +- crates/core_arch/src/x86/avx512cd.rs | 2 +- crates/core_arch/src/x86/avx512dq.rs | 2 +- crates/core_arch/src/x86/avx512f.rs | 2 +- crates/core_arch/src/x86/avx512ifma.rs | 2 +- crates/core_arch/src/x86/avx512vbmi.rs | 2 +- crates/core_arch/src/x86/avx512vbmi2.rs | 2 +- crates/core_arch/src/x86/avx512vnni.rs | 2 +- crates/core_arch/src/x86/avx512vp2intersect.rs | 2 +- crates/core_arch/src/x86/avxneconvert.rs | 2 +- crates/core_arch/src/x86/bmi1.rs | 2 +- crates/core_arch/src/x86/bmi2.rs | 2 +- crates/core_arch/src/x86/fxsr.rs | 2 +- crates/core_arch/src/x86/gfni.rs | 2 +- crates/core_arch/src/x86/pclmulqdq.rs | 2 +- crates/core_arch/src/x86/rtm.rs | 2 +- crates/core_arch/src/x86/sha.rs | 2 +- crates/core_arch/src/x86/sse.rs | 2 +- crates/core_arch/src/x86/sse2.rs | 2 +- crates/core_arch/src/x86/sse3.rs | 2 +- crates/core_arch/src/x86/sse41.rs | 2 +- crates/core_arch/src/x86/sse42.rs | 2 +- crates/core_arch/src/x86/sse4a.rs | 2 +- crates/core_arch/src/x86/ssse3.rs | 2 +- crates/core_arch/src/x86/tbm.rs | 2 +- crates/core_arch/src/x86/vaes.rs | 2 +- crates/core_arch/src/x86/vpclmulqdq.rs | 2 +- crates/core_arch/src/x86/xsave.rs | 2 +- crates/core_arch/src/x86_64/amx.rs | 2 +- crates/core_arch/src/x86_64/avx512f.rs | 2 +- crates/core_arch/src/x86_64/avx512fp16.rs | 2 +- crates/core_arch/src/x86_64/bmi.rs | 2 +- crates/core_arch/src/x86_64/bmi2.rs | 2 +- crates/core_arch/src/x86_64/fxsr.rs | 2 +- crates/core_arch/src/x86_64/sse.rs | 2 +- crates/core_arch/src/x86_64/sse2.rs | 2 +- crates/core_arch/src/x86_64/sse42.rs | 2 +- crates/core_arch/src/x86_64/tbm.rs | 2 +- crates/core_arch/src/x86_64/xsave.rs | 2 +- 51 files changed, 52 insertions(+), 52 deletions(-) diff --git a/crates/core_arch/src/aarch64/sve/mod.rs b/crates/core_arch/src/aarch64/sve/mod.rs index a3f70ab61c..c6d5d57aca 100644 --- a/crates/core_arch/src/aarch64/sve/mod.rs +++ b/crates/core_arch/src/aarch64/sve/mod.rs @@ -130,7 +130,7 @@ macro_rules! impl_internal_sve_predicate { #[target_feature(enable = "sve")] unsafe fn sve_into(self) -> svbool_t { #[allow(improper_ctypes)] - unsafe extern "C" { + unsafe extern "unadjusted" { #[cfg_attr( target_arch = "aarch64", link_name = concat!("llvm.aarch64.sve.convert.to.svbool.nxv", $elt, "i1") @@ -147,7 +147,7 @@ macro_rules! impl_internal_sve_predicate { #[target_feature(enable = "sve")] unsafe fn sve_into(self) -> $name { #[allow(improper_ctypes)] - unsafe extern "C" { + unsafe extern "unadjusted" { #[cfg_attr( target_arch = "aarch64", link_name = concat!("llvm.aarch64.sve.convert.from.svbool.nxv", $elt, "i1") diff --git a/crates/core_arch/src/mips/msa.rs b/crates/core_arch/src/mips/msa.rs index bc601baef9..dab64319cc 100644 --- a/crates/core_arch/src/mips/msa.rs +++ b/crates/core_arch/src/mips/msa.rs @@ -45,7 +45,7 @@ types! { } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.mips.add.a.b"] fn msa_add_a_b(a: v16i8, b: v16i8) -> v16i8; #[link_name = "llvm.mips.add.a.h"] diff --git a/crates/core_arch/src/nvptx/mod.rs b/crates/core_arch/src/nvptx/mod.rs index 84d8810320..d22f3a25bf 100644 --- a/crates/core_arch/src/nvptx/mod.rs +++ b/crates/core_arch/src/nvptx/mod.rs @@ -19,7 +19,7 @@ mod packed; pub use packed::*; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.nvvm.barrier.cta.sync.aligned.all"] fn syncthreads(a: u32) -> (); #[link_name = "llvm.nvvm.read.ptx.sreg.ntid.x"] diff --git a/crates/core_arch/src/nvptx/packed.rs b/crates/core_arch/src/nvptx/packed.rs index 1c7e81268f..7e6c14bc99 100644 --- a/crates/core_arch/src/nvptx/packed.rs +++ b/crates/core_arch/src/nvptx/packed.rs @@ -7,7 +7,7 @@ use crate::intrinsics::simd::*; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.minimum.v2f16"] fn llvm_f16x2_minimum(a: f16x2, b: f16x2) -> f16x2; #[link_name = "llvm.maximum.v2f16"] diff --git a/crates/core_arch/src/powerpc/altivec.rs b/crates/core_arch/src/powerpc/altivec.rs index 78ec39f91f..ccfed88a93 100644 --- a/crates/core_arch/src/powerpc/altivec.rs +++ b/crates/core_arch/src/powerpc/altivec.rs @@ -96,7 +96,7 @@ impl From for m32x4 { } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.ppc.altivec.lvx"] fn lvx(p: *const i8) -> vector_unsigned_int; diff --git a/crates/core_arch/src/powerpc/vsx.rs b/crates/core_arch/src/powerpc/vsx.rs index 4a7b561a20..60cb2ad44c 100644 --- a/crates/core_arch/src/powerpc/vsx.rs +++ b/crates/core_arch/src/powerpc/vsx.rs @@ -52,7 +52,7 @@ impl From for m64x2 { } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.ppc.altivec.vperm"] fn vperm( a: vector_signed_int, diff --git a/crates/core_arch/src/powerpc64/vsx.rs b/crates/core_arch/src/powerpc64/vsx.rs index 7b42be8653..9032e77959 100644 --- a/crates/core_arch/src/powerpc64/vsx.rs +++ b/crates/core_arch/src/powerpc64/vsx.rs @@ -17,7 +17,7 @@ use stdarch_test::assert_instr; use crate::mem::transmute; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.ppc.vsx.lxvl"] fn lxvl(a: *const u8, l: usize) -> vector_signed_int; diff --git a/crates/core_arch/src/x86/aes.rs b/crates/core_arch/src/x86/aes.rs index d07ab4dc2a..ecb09a6a90 100644 --- a/crates/core_arch/src/x86/aes.rs +++ b/crates/core_arch/src/x86/aes.rs @@ -13,7 +13,7 @@ use crate::core_arch::x86::__m128i; use stdarch_test::assert_instr; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.aesni.aesdec"] fn aesdec(a: __m128i, round_key: __m128i) -> __m128i; #[link_name = "llvm.x86.aesni.aesdeclast"] diff --git a/crates/core_arch/src/x86/avx.rs b/crates/core_arch/src/x86/avx.rs index ef434205b5..f5ddff6457 100644 --- a/crates/core_arch/src/x86/avx.rs +++ b/crates/core_arch/src/x86/avx.rs @@ -3292,7 +3292,7 @@ pub const fn _mm256_cvtss_f32(a: __m256) -> f32 { // LLVM intrinsics used in the above functions #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.avx.round.pd.256"] fn roundpd256(a: __m256d, b: i32) -> __m256d; #[link_name = "llvm.x86.avx.round.ps.256"] diff --git a/crates/core_arch/src/x86/avx2.rs b/crates/core_arch/src/x86/avx2.rs index b49ad9522a..6925ba8e27 100644 --- a/crates/core_arch/src/x86/avx2.rs +++ b/crates/core_arch/src/x86/avx2.rs @@ -3906,7 +3906,7 @@ pub const fn _mm256_extract_epi16(a: __m256i) -> i32 { } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.avx2.pmadd.wd"] fn pmaddwd(a: i16x16, b: i16x16) -> i32x8; #[link_name = "llvm.x86.avx2.pmadd.ub.sw"] diff --git a/crates/core_arch/src/x86/avx512bf16.rs b/crates/core_arch/src/x86/avx512bf16.rs index 8d944f5ba8..230a4f3754 100644 --- a/crates/core_arch/src/x86/avx512bf16.rs +++ b/crates/core_arch/src/x86/avx512bf16.rs @@ -9,7 +9,7 @@ use crate::intrinsics::simd::*; use stdarch_test::assert_instr; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.avx512bf16.cvtne2ps2bf16.128"] fn cvtne2ps2bf16(a: f32x4, b: f32x4) -> i16x8; #[link_name = "llvm.x86.avx512bf16.cvtne2ps2bf16.256"] diff --git a/crates/core_arch/src/x86/avx512bitalg.rs b/crates/core_arch/src/x86/avx512bitalg.rs index dd211854af..344130fb6e 100644 --- a/crates/core_arch/src/x86/avx512bitalg.rs +++ b/crates/core_arch/src/x86/avx512bitalg.rs @@ -27,7 +27,7 @@ use crate::mem::transmute; use stdarch_test::assert_instr; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.avx512.vpshufbitqmb.512"] fn bitshuffle_512(data: i8x64, indices: i8x64) -> __mmask64; #[link_name = "llvm.x86.avx512.vpshufbitqmb.256"] diff --git a/crates/core_arch/src/x86/avx512bw.rs b/crates/core_arch/src/x86/avx512bw.rs index 659d6c3be8..50d57b4964 100644 --- a/crates/core_arch/src/x86/avx512bw.rs +++ b/crates/core_arch/src/x86/avx512bw.rs @@ -12764,7 +12764,7 @@ pub unsafe fn _mm_mask_cvtusepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.avx512.pmul.hr.sw.512"] fn vpmulhrsw(a: i16x32, b: i16x32) -> i16x32; diff --git a/crates/core_arch/src/x86/avx512cd.rs b/crates/core_arch/src/x86/avx512cd.rs index 4082433e70..378bc3617e 100644 --- a/crates/core_arch/src/x86/avx512cd.rs +++ b/crates/core_arch/src/x86/avx512cd.rs @@ -563,7 +563,7 @@ pub const fn _mm_maskz_lzcnt_epi64(k: __mmask8, a: __m128i) -> __m128i { } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.avx512.conflict.d.512"] fn vpconflictd(a: i32x16) -> i32x16; #[link_name = "llvm.x86.avx512.conflict.d.256"] diff --git a/crates/core_arch/src/x86/avx512dq.rs b/crates/core_arch/src/x86/avx512dq.rs index 0b322c8b83..5fe40b7541 100644 --- a/crates/core_arch/src/x86/avx512dq.rs +++ b/crates/core_arch/src/x86/avx512dq.rs @@ -7235,7 +7235,7 @@ pub fn _mm_mask_fpclass_ss_mask(k1: __mmask8, a: __m128) -> __m } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.avx512.sitofp.round.v2f64.v2i64"] fn vcvtqq2pd_128(a: i64x2, rounding: i32) -> f64x2; #[link_name = "llvm.x86.avx512.sitofp.round.v4f64.v4i64"] diff --git a/crates/core_arch/src/x86/avx512f.rs b/crates/core_arch/src/x86/avx512f.rs index 66ea63b674..225e447cea 100644 --- a/crates/core_arch/src/x86/avx512f.rs +++ b/crates/core_arch/src/x86/avx512f.rs @@ -44215,7 +44215,7 @@ pub const _MM_PERM_DDDC: _MM_PERM_ENUM = 0xFE; pub const _MM_PERM_DDDD: _MM_PERM_ENUM = 0xFF; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.avx512.sqrt.ps.512"] fn vsqrtps(a: f32x16, rounding: i32) -> f32x16; #[link_name = "llvm.x86.avx512.sqrt.pd.512"] diff --git a/crates/core_arch/src/x86/avx512ifma.rs b/crates/core_arch/src/x86/avx512ifma.rs index 5ce28565d1..262215e62c 100644 --- a/crates/core_arch/src/x86/avx512ifma.rs +++ b/crates/core_arch/src/x86/avx512ifma.rs @@ -347,7 +347,7 @@ pub fn _mm_maskz_madd52lo_epu64(k: __mmask8, a: __m128i, b: __m128i, c: __m128i) } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.avx512.vpmadd52l.uq.128"] fn vpmadd52luq_128(z: __m128i, x: __m128i, y: __m128i) -> __m128i; #[link_name = "llvm.x86.avx512.vpmadd52h.uq.128"] diff --git a/crates/core_arch/src/x86/avx512vbmi.rs b/crates/core_arch/src/x86/avx512vbmi.rs index d9ad14ef00..141a8c8aec 100644 --- a/crates/core_arch/src/x86/avx512vbmi.rs +++ b/crates/core_arch/src/x86/avx512vbmi.rs @@ -453,7 +453,7 @@ pub fn _mm_maskz_multishift_epi64_epi8(k: __mmask16, a: __m128i, b: __m128i) -> } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.avx512.vpermi2var.qi.512"] fn vpermi2b(a: i8x64, idx: i8x64, b: i8x64) -> i8x64; #[link_name = "llvm.x86.avx512.vpermi2var.qi.256"] diff --git a/crates/core_arch/src/x86/avx512vbmi2.rs b/crates/core_arch/src/x86/avx512vbmi2.rs index 78a50b90c8..0a4accc2f7 100644 --- a/crates/core_arch/src/x86/avx512vbmi2.rs +++ b/crates/core_arch/src/x86/avx512vbmi2.rs @@ -2383,7 +2383,7 @@ pub const fn _mm_maskz_shrdi_epi16( } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.avx512.mask.compress.store.w.512"] fn vcompressstorew(mem: *mut i8, data: i16x32, mask: u32); #[link_name = "llvm.x86.avx512.mask.compress.store.w.256"] diff --git a/crates/core_arch/src/x86/avx512vnni.rs b/crates/core_arch/src/x86/avx512vnni.rs index 8cd8764f24..b9084aec59 100644 --- a/crates/core_arch/src/x86/avx512vnni.rs +++ b/crates/core_arch/src/x86/avx512vnni.rs @@ -873,7 +873,7 @@ pub fn _mm256_dpwuuds_epi32(src: __m256i, a: __m256i, b: __m256i) -> __m256i { } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.avx512.vpdpwssd.512"] fn vpdpwssd(src: i32x16, a: i16x32, b: i16x32) -> i32x16; #[link_name = "llvm.x86.avx512.vpdpwssd.256"] diff --git a/crates/core_arch/src/x86/avx512vp2intersect.rs b/crates/core_arch/src/x86/avx512vp2intersect.rs index 4dd7412e9e..30fcfce63c 100644 --- a/crates/core_arch/src/x86/avx512vp2intersect.rs +++ b/crates/core_arch/src/x86/avx512vp2intersect.rs @@ -110,7 +110,7 @@ pub unsafe fn _mm512_2intersect_epi64( } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.avx512.vp2intersect.d.128"] fn vp2intersectd_128(a: i32x4, b: i32x4) -> (u8, u8); #[link_name = "llvm.x86.avx512.vp2intersect.q.128"] diff --git a/crates/core_arch/src/x86/avxneconvert.rs b/crates/core_arch/src/x86/avxneconvert.rs index 861213eb42..b17d3de48b 100644 --- a/crates/core_arch/src/x86/avxneconvert.rs +++ b/crates/core_arch/src/x86/avxneconvert.rs @@ -176,7 +176,7 @@ pub fn _mm256_cvtneps_avx_pbh(a: __m256) -> __m128bh { } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.vbcstnebf162ps128"] fn bcstnebf162ps_128(a: *const bf16) -> __m128; #[link_name = "llvm.x86.vbcstnebf162ps256"] diff --git a/crates/core_arch/src/x86/bmi1.rs b/crates/core_arch/src/x86/bmi1.rs index 432051abd1..21da03667f 100644 --- a/crates/core_arch/src/x86/bmi1.rs +++ b/crates/core_arch/src/x86/bmi1.rs @@ -131,7 +131,7 @@ pub const fn _mm_tzcnt_32(x: u32) -> i32 { x.trailing_zeros() as i32 } -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.bmi.bextr.32"] fn x86_bmi_bextr_32(x: u32, y: u32) -> u32; } diff --git a/crates/core_arch/src/x86/bmi2.rs b/crates/core_arch/src/x86/bmi2.rs index 5320640d96..dae4133d63 100644 --- a/crates/core_arch/src/x86/bmi2.rs +++ b/crates/core_arch/src/x86/bmi2.rs @@ -67,7 +67,7 @@ pub fn _pext_u32(a: u32, mask: u32) -> u32 { unsafe { x86_bmi2_pext_32(a, mask) } } -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.bmi.bzhi.32"] fn x86_bmi2_bzhi_32(x: u32, y: u32) -> u32; #[link_name = "llvm.x86.bmi.pdep.32"] diff --git a/crates/core_arch/src/x86/fxsr.rs b/crates/core_arch/src/x86/fxsr.rs index 08619efe7c..74a95e192a 100644 --- a/crates/core_arch/src/x86/fxsr.rs +++ b/crates/core_arch/src/x86/fxsr.rs @@ -4,7 +4,7 @@ use stdarch_test::assert_instr; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.fxsave"] fn fxsave(p: *mut u8); #[link_name = "llvm.x86.fxrstor"] diff --git a/crates/core_arch/src/x86/gfni.rs b/crates/core_arch/src/x86/gfni.rs index e9ee27a7b8..8cebd04a4b 100644 --- a/crates/core_arch/src/x86/gfni.rs +++ b/crates/core_arch/src/x86/gfni.rs @@ -23,7 +23,7 @@ use crate::mem::transmute; use stdarch_test::assert_instr; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.vgf2p8affineinvqb.512"] fn vgf2p8affineinvqb_512(x: i8x64, a: i8x64, imm8: u8) -> i8x64; #[link_name = "llvm.x86.vgf2p8affineinvqb.256"] diff --git a/crates/core_arch/src/x86/pclmulqdq.rs b/crates/core_arch/src/x86/pclmulqdq.rs index 0f2769257f..454785c5e0 100644 --- a/crates/core_arch/src/x86/pclmulqdq.rs +++ b/crates/core_arch/src/x86/pclmulqdq.rs @@ -11,7 +11,7 @@ use crate::core_arch::x86::__m128i; use stdarch_test::assert_instr; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.pclmulqdq"] fn pclmulqdq(a: __m128i, round_key: __m128i, imm8: u8) -> __m128i; } diff --git a/crates/core_arch/src/x86/rtm.rs b/crates/core_arch/src/x86/rtm.rs index c88bd6592d..f09b95d76b 100644 --- a/crates/core_arch/src/x86/rtm.rs +++ b/crates/core_arch/src/x86/rtm.rs @@ -16,7 +16,7 @@ #[cfg(test)] use stdarch_test::assert_instr; -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.xbegin"] fn x86_xbegin() -> i32; #[link_name = "llvm.x86.xend"] diff --git a/crates/core_arch/src/x86/sha.rs b/crates/core_arch/src/x86/sha.rs index f8a3295d19..f7d8c3c0b2 100644 --- a/crates/core_arch/src/x86/sha.rs +++ b/crates/core_arch/src/x86/sha.rs @@ -1,7 +1,7 @@ use crate::core_arch::{simd::*, x86::*}; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.sha1msg1"] fn sha1msg1(a: i32x4, b: i32x4) -> i32x4; #[link_name = "llvm.x86.sha1msg2"] diff --git a/crates/core_arch/src/x86/sse.rs b/crates/core_arch/src/x86/sse.rs index afaaccfbf2..e31175d48c 100644 --- a/crates/core_arch/src/x86/sse.rs +++ b/crates/core_arch/src/x86/sse.rs @@ -1987,7 +1987,7 @@ pub const fn _MM_TRANSPOSE4_PS( } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.sse.rcp.ss"] fn rcpss(a: __m128) -> __m128; #[link_name = "llvm.x86.sse.rcp.ps"] diff --git a/crates/core_arch/src/x86/sse2.rs b/crates/core_arch/src/x86/sse2.rs index 1f97f3c69d..983d7ff23d 100644 --- a/crates/core_arch/src/x86/sse2.rs +++ b/crates/core_arch/src/x86/sse2.rs @@ -3236,7 +3236,7 @@ pub const fn _mm_unpacklo_pd(a: __m128d, b: __m128d) -> __m128d { } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.sse2.pause"] fn pause(); #[link_name = "llvm.x86.sse2.clflush"] diff --git a/crates/core_arch/src/x86/sse3.rs b/crates/core_arch/src/x86/sse3.rs index e4c7570254..2f6fa35cb6 100644 --- a/crates/core_arch/src/x86/sse3.rs +++ b/crates/core_arch/src/x86/sse3.rs @@ -178,7 +178,7 @@ pub const fn _mm_moveldup_ps(a: __m128) -> __m128 { } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.sse3.ldu.dq"] fn lddqu(mem_addr: *const i8) -> i8x16; } diff --git a/crates/core_arch/src/x86/sse41.rs b/crates/core_arch/src/x86/sse41.rs index 4ebf7d3bd3..d981166ff0 100644 --- a/crates/core_arch/src/x86/sse41.rs +++ b/crates/core_arch/src/x86/sse41.rs @@ -1181,7 +1181,7 @@ pub unsafe fn _mm_stream_load_si128(mem_addr: *const __m128i) -> __m128i { } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.sse41.insertps"] fn insertps(a: __m128, b: __m128, imm8: u8) -> __m128; #[link_name = "llvm.x86.sse41.dppd"] diff --git a/crates/core_arch/src/x86/sse42.rs b/crates/core_arch/src/x86/sse42.rs index 55e2259263..0534fe9379 100644 --- a/crates/core_arch/src/x86/sse42.rs +++ b/crates/core_arch/src/x86/sse42.rs @@ -569,7 +569,7 @@ pub const fn _mm_cmpgt_epi64(a: __m128i, b: __m128i) -> __m128i { } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { // SSE 4.2 string and text comparison ops #[link_name = "llvm.x86.sse42.pcmpestrm128"] fn pcmpestrm128(a: i8x16, la: i32, b: i8x16, lb: i32, imm8: i8) -> u8x16; diff --git a/crates/core_arch/src/x86/sse4a.rs b/crates/core_arch/src/x86/sse4a.rs index f36b879a03..14e9b56dcb 100644 --- a/crates/core_arch/src/x86/sse4a.rs +++ b/crates/core_arch/src/x86/sse4a.rs @@ -6,7 +6,7 @@ use crate::core_arch::{simd::*, x86::*}; use stdarch_test::assert_instr; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.sse4a.extrq"] fn extrq(x: i64x2, y: i8x16) -> i64x2; #[link_name = "llvm.x86.sse4a.extrqi"] diff --git a/crates/core_arch/src/x86/ssse3.rs b/crates/core_arch/src/x86/ssse3.rs index 1d7a97944a..b0152c8159 100644 --- a/crates/core_arch/src/x86/ssse3.rs +++ b/crates/core_arch/src/x86/ssse3.rs @@ -345,7 +345,7 @@ pub fn _mm_sign_epi32(a: __m128i, b: __m128i) -> __m128i { } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.ssse3.pshuf.b.128"] fn pshufb128(a: u8x16, b: u8x16) -> u8x16; diff --git a/crates/core_arch/src/x86/tbm.rs b/crates/core_arch/src/x86/tbm.rs index 0ba4572dcd..2a44579408 100644 --- a/crates/core_arch/src/x86/tbm.rs +++ b/crates/core_arch/src/x86/tbm.rs @@ -13,7 +13,7 @@ #[cfg(test)] use stdarch_test::assert_instr; -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.tbm.bextri.u32"] fn bextri_u32(a: u32, control: u32) -> u32; } diff --git a/crates/core_arch/src/x86/vaes.rs b/crates/core_arch/src/x86/vaes.rs index 864b1d56d1..72dd387f90 100644 --- a/crates/core_arch/src/x86/vaes.rs +++ b/crates/core_arch/src/x86/vaes.rs @@ -14,7 +14,7 @@ use crate::core_arch::x86::__m512i; use stdarch_test::assert_instr; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.aesni.aesenc.256"] fn aesenc_256(a: __m256i, round_key: __m256i) -> __m256i; #[link_name = "llvm.x86.aesni.aesenclast.256"] diff --git a/crates/core_arch/src/x86/vpclmulqdq.rs b/crates/core_arch/src/x86/vpclmulqdq.rs index ad44e59f3a..4c39ae6ee7 100644 --- a/crates/core_arch/src/x86/vpclmulqdq.rs +++ b/crates/core_arch/src/x86/vpclmulqdq.rs @@ -12,7 +12,7 @@ use crate::core_arch::x86::__m512i; use stdarch_test::assert_instr; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.pclmulqdq.256"] fn pclmulqdq_256(a: __m256i, round_key: __m256i, imm8: u8) -> __m256i; #[link_name = "llvm.x86.pclmulqdq.512"] diff --git a/crates/core_arch/src/x86/xsave.rs b/crates/core_arch/src/x86/xsave.rs index e22d3580ff..395e2c64be 100644 --- a/crates/core_arch/src/x86/xsave.rs +++ b/crates/core_arch/src/x86/xsave.rs @@ -5,7 +5,7 @@ use stdarch_test::assert_instr; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.xsave"] fn xsave(p: *mut u8, hi: u32, lo: u32); #[link_name = "llvm.x86.xrstor"] diff --git a/crates/core_arch/src/x86_64/amx.rs b/crates/core_arch/src/x86_64/amx.rs index 08585d2067..2e46e50f9f 100644 --- a/crates/core_arch/src/x86_64/amx.rs +++ b/crates/core_arch/src/x86_64/amx.rs @@ -595,7 +595,7 @@ pub unsafe fn _tile_movrowi() -> __m512i { } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.ldtilecfg"] fn ldtilecfg(mem_addr: *const u8); #[link_name = "llvm.x86.sttilecfg"] diff --git a/crates/core_arch/src/x86_64/avx512f.rs b/crates/core_arch/src/x86_64/avx512f.rs index 0fd9b09363..e26bfc5c1c 100644 --- a/crates/core_arch/src/x86_64/avx512f.rs +++ b/crates/core_arch/src/x86_64/avx512f.rs @@ -527,7 +527,7 @@ pub fn _mm_cvtt_roundss_u64(a: __m128) -> u64 { } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.avx512.vcvtss2si64"] fn vcvtss2si64(a: f32x4, rounding: i32) -> i64; #[link_name = "llvm.x86.avx512.vcvtss2usi64"] diff --git a/crates/core_arch/src/x86_64/avx512fp16.rs b/crates/core_arch/src/x86_64/avx512fp16.rs index 2a511328bb..5baf1c9036 100644 --- a/crates/core_arch/src/x86_64/avx512fp16.rs +++ b/crates/core_arch/src/x86_64/avx512fp16.rs @@ -211,7 +211,7 @@ pub fn _mm_cvtt_roundsh_u64(a: __m128h) -> u64 { } #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.avx512fp16.vcvtsi642sh"] fn vcvtsi642sh(a: __m128h, b: i64, rounding: i32) -> __m128h; #[link_name = "llvm.x86.avx512fp16.vcvtusi642sh"] diff --git a/crates/core_arch/src/x86_64/bmi.rs b/crates/core_arch/src/x86_64/bmi.rs index 8d2b22089a..f082953f9d 100644 --- a/crates/core_arch/src/x86_64/bmi.rs +++ b/crates/core_arch/src/x86_64/bmi.rs @@ -122,7 +122,7 @@ pub const fn _mm_tzcnt_64(x: u64) -> i64 { x.trailing_zeros() as i64 } -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.bmi.bextr.64"] fn x86_bmi_bextr_64(x: u64, y: u64) -> u64; } diff --git a/crates/core_arch/src/x86_64/bmi2.rs b/crates/core_arch/src/x86_64/bmi2.rs index 6151eee8bd..81ab9e05ad 100644 --- a/crates/core_arch/src/x86_64/bmi2.rs +++ b/crates/core_arch/src/x86_64/bmi2.rs @@ -69,7 +69,7 @@ pub fn _pext_u64(a: u64, mask: u64) -> u64 { unsafe { x86_bmi2_pext_64(a, mask) } } -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.bmi.bzhi.64"] fn x86_bmi2_bzhi_64(x: u64, y: u64) -> u64; #[link_name = "llvm.x86.bmi.pdep.64"] diff --git a/crates/core_arch/src/x86_64/fxsr.rs b/crates/core_arch/src/x86_64/fxsr.rs index 28bf195116..f50dacb4be 100644 --- a/crates/core_arch/src/x86_64/fxsr.rs +++ b/crates/core_arch/src/x86_64/fxsr.rs @@ -4,7 +4,7 @@ use stdarch_test::assert_instr; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.fxsave64"] fn fxsave64(p: *mut u8); #[link_name = "llvm.x86.fxrstor64"] diff --git a/crates/core_arch/src/x86_64/sse.rs b/crates/core_arch/src/x86_64/sse.rs index 81e1070b55..521652c126 100644 --- a/crates/core_arch/src/x86_64/sse.rs +++ b/crates/core_arch/src/x86_64/sse.rs @@ -6,7 +6,7 @@ use crate::core_arch::x86::*; use stdarch_test::assert_instr; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.sse.cvtss2si64"] fn cvtss2si64(a: __m128) -> i64; #[link_name = "llvm.x86.sse.cvttss2si64"] diff --git a/crates/core_arch/src/x86_64/sse2.rs b/crates/core_arch/src/x86_64/sse2.rs index 08dabf053d..c4768cedbf 100644 --- a/crates/core_arch/src/x86_64/sse2.rs +++ b/crates/core_arch/src/x86_64/sse2.rs @@ -6,7 +6,7 @@ use crate::core_arch::x86::*; use stdarch_test::assert_instr; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.sse2.cvtsd2si64"] fn cvtsd2si64(a: __m128d) -> i64; #[link_name = "llvm.x86.sse2.cvttsd2si64"] diff --git a/crates/core_arch/src/x86_64/sse42.rs b/crates/core_arch/src/x86_64/sse42.rs index cd32c149af..307e6167aa 100644 --- a/crates/core_arch/src/x86_64/sse42.rs +++ b/crates/core_arch/src/x86_64/sse42.rs @@ -4,7 +4,7 @@ use stdarch_test::assert_instr; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.sse42.crc32.64.64"] fn crc32_64_64(crc: u64, v: u64) -> u64; } diff --git a/crates/core_arch/src/x86_64/tbm.rs b/crates/core_arch/src/x86_64/tbm.rs index fe12538b07..b4ec50a66f 100644 --- a/crates/core_arch/src/x86_64/tbm.rs +++ b/crates/core_arch/src/x86_64/tbm.rs @@ -13,7 +13,7 @@ #[cfg(test)] use stdarch_test::assert_instr; -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.tbm.bextri.u64"] fn bextri_u64(a: u64, control: u64) -> u64; } diff --git a/crates/core_arch/src/x86_64/xsave.rs b/crates/core_arch/src/x86_64/xsave.rs index 30a7123315..c307d40073 100644 --- a/crates/core_arch/src/x86_64/xsave.rs +++ b/crates/core_arch/src/x86_64/xsave.rs @@ -6,7 +6,7 @@ use stdarch_test::assert_instr; #[allow(improper_ctypes)] -unsafe extern "C" { +unsafe extern "unadjusted" { #[link_name = "llvm.x86.xsave64"] fn xsave64(p: *mut u8, hi: u32, lo: u32); #[link_name = "llvm.x86.xrstor64"]