From d5c0fbe8c5db0fc0d69424ccfde698c1cf355d72 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 31 Aug 2026 17:57:14 +0800 Subject: [PATCH 01/33] latx: simplify AVX floating-point translation Write floating-point arithmetic, FMA, comparison and conversion results directly to the destination where source registers can be preserved. Remove redundant temporary copies and use vector-register alias checks in narrowing conversions. (cherry picked from commit c8d4ac52e366d29664d28294bd881f6c0c1b1b45) Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx-cmp.c | 256 +++++------------- target/i386/latx/translator/tr-avx-cvt.c | 81 +++--- target/i386/latx/translator/tr-avx.c | 122 +++++---- .../integration/registrations/avx/meson.build | 10 + .../integration/test-avx-conversion-alias.py | 17 ++ tests/integration/vcvtdq2pd-ymm-alias.asm | 64 +++++ tests/integration/vcvtpd-register-alias.asm | 81 ++++++ 7 files changed, 350 insertions(+), 281 deletions(-) create mode 100755 tests/integration/test-avx-conversion-alias.py create mode 100644 tests/integration/vcvtdq2pd-ymm-alias.asm create mode 100644 tests/integration/vcvtpd-register-alias.asm diff --git a/target/i386/latx/translator/tr-avx-cmp.c b/target/i386/latx/translator/tr-avx-cmp.c index 5fb062c777..f77dea6f59 100644 --- a/target/i386/latx/translator/tr-avx-cmp.c +++ b/target/i386/latx/translator/tr-avx-cmp.c @@ -198,9 +198,7 @@ bool translate_vcmpeqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_EQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_EQ); set_high128_xreg_to_zero(dest); } return true; @@ -224,9 +222,7 @@ bool translate_vcmpltpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_LT); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_LT); set_high128_xreg_to_zero(dest); } return true; @@ -250,9 +246,7 @@ bool translate_vcmplepd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_LE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_LE); set_high128_xreg_to_zero(dest); } return true; @@ -276,9 +270,7 @@ bool translate_vcmpunordpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_UNORD); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_UNORD); set_high128_xreg_to_zero(dest); } return true; @@ -302,9 +294,7 @@ bool translate_vcmpneqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NEQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_NEQ); set_high128_xreg_to_zero(dest); } return true; @@ -329,9 +319,7 @@ bool translate_vcmpnltpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_NLT); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src2, src1, X86_FCMP_COND_NLT); set_high128_xreg_to_zero(dest); } return true; @@ -355,9 +343,7 @@ bool translate_vcmpnlepd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_NLE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src2, src1, X86_FCMP_COND_NLE); set_high128_xreg_to_zero(dest); } return true; @@ -381,9 +367,7 @@ bool translate_vcmpordpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_ORD); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_ORD); set_high128_xreg_to_zero(dest); } return true; @@ -407,9 +391,7 @@ bool translate_vcmpeq_uqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_EQ_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_EQ_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -433,9 +415,7 @@ bool translate_vcmpngepd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NGE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_NGE); set_high128_xreg_to_zero(dest); } return true; @@ -459,9 +439,7 @@ bool translate_vcmpngtpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NGT); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_NGT); set_high128_xreg_to_zero(dest); } return true; @@ -485,9 +463,7 @@ bool translate_vcmpfalsepd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_FALSE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_FALSE); set_high128_xreg_to_zero(dest); } return true; @@ -511,9 +487,7 @@ bool translate_vcmpneq_oqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NEQ_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_NEQ_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -537,9 +511,7 @@ bool translate_vcmpgepd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_GE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src2, src1, X86_FCMP_COND_GE); set_high128_xreg_to_zero(dest); } return true; @@ -563,9 +535,7 @@ bool translate_vcmpgtpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_GT); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src2, src1, X86_FCMP_COND_GT); set_high128_xreg_to_zero(dest); } return true; @@ -591,10 +561,8 @@ bool translate_vcmptruepd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_TRUE); - la_xvori_b(temp, temp, 0xff); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_TRUE); + la_xvori_b(dest, dest, 0xff); set_high128_xreg_to_zero(dest); } return true; @@ -618,9 +586,7 @@ bool translate_vcmpeq_ospd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_EQ_OS); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_EQ_OS); set_high128_xreg_to_zero(dest); } return true; @@ -644,9 +610,7 @@ bool translate_vcmplt_oqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_LT_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_LT_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -670,9 +634,7 @@ bool translate_vcmple_oqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_LE_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_LE_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -696,9 +658,7 @@ bool translate_vcmpunord_spd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_UNORD_S); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_UNORD_S); set_high128_xreg_to_zero(dest); } return true; @@ -722,9 +682,7 @@ bool translate_vcmpneq_uspd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NEQ_US); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_NEQ_US); set_high128_xreg_to_zero(dest); } return true; @@ -748,9 +706,7 @@ bool translate_vcmpnlt_uqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_NLT_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src2, src1, X86_FCMP_COND_NLT_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -774,9 +730,7 @@ bool translate_vcmpnle_uqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_NLE_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src2, src1, X86_FCMP_COND_NLE_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -800,9 +754,7 @@ bool translate_vcmpord_spd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_ORD_S); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_ORD_S); set_high128_xreg_to_zero(dest); } return true; @@ -826,9 +778,7 @@ bool translate_vcmpeq_uspd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_EQ_US); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_EQ_US); set_high128_xreg_to_zero(dest); } return true; @@ -852,9 +802,7 @@ bool translate_vcmpnge_uqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NGE_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_NGE_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -878,9 +826,7 @@ bool translate_vcmpngt_uqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NGT_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_NGT_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -904,9 +850,7 @@ bool translate_vcmpfalse_ospd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_FALSE_OS); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_FALSE_OS); set_high128_xreg_to_zero(dest); } return true; @@ -930,9 +874,7 @@ bool translate_vcmpneq_ospd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NEQ_OS); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_NEQ_OS); set_high128_xreg_to_zero(dest); } return true; @@ -956,9 +898,7 @@ bool translate_vcmpge_oqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_GE_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src2, src1, X86_FCMP_COND_GE_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -982,9 +922,7 @@ bool translate_vcmpgt_oqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_GT_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src2, src1, X86_FCMP_COND_GT_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -1010,10 +948,8 @@ bool translate_vcmptrue_uspd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_TRUE_US); - la_xvori_b(temp, temp, 0xff); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_TRUE_US); + la_xvori_b(dest, dest, 0xff); set_high128_xreg_to_zero(dest); } return true; @@ -1168,9 +1104,7 @@ bool translate_vcmpleps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_LE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_LE); set_high128_xreg_to_zero(dest); } return true; @@ -1194,9 +1128,7 @@ bool translate_vcmpunordps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_UNORD); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_UNORD); set_high128_xreg_to_zero(dest); } return true; @@ -1220,9 +1152,7 @@ bool translate_vcmpneqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NEQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_NEQ); set_high128_xreg_to_zero(dest); } return true; @@ -1247,9 +1177,7 @@ bool translate_vcmpnltps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_NLT); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src2, src1, X86_FCMP_COND_NLT); set_high128_xreg_to_zero(dest); } return true; @@ -1273,9 +1201,7 @@ bool translate_vcmpnleps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_NLE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src2, src1, X86_FCMP_COND_NLE); set_high128_xreg_to_zero(dest); } return true; @@ -1299,9 +1225,7 @@ bool translate_vcmpordps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_ORD); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_ORD); set_high128_xreg_to_zero(dest); } return true; @@ -1325,9 +1249,7 @@ bool translate_vcmpeq_uqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_EQ_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_EQ_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -1351,9 +1273,7 @@ bool translate_vcmpngeps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NGE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_NGE); set_high128_xreg_to_zero(dest); } return true; @@ -1377,9 +1297,7 @@ bool translate_vcmpngtps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NGT); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_NGT); set_high128_xreg_to_zero(dest); } return true; @@ -1403,9 +1321,7 @@ bool translate_vcmpfalseps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_FALSE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_FALSE); set_high128_xreg_to_zero(dest); } return true; @@ -1429,9 +1345,7 @@ bool translate_vcmpneq_oqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NEQ_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_NEQ_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -1455,9 +1369,7 @@ bool translate_vcmpgeps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_GE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src2, src1, X86_FCMP_COND_GE); set_high128_xreg_to_zero(dest); } return true; @@ -1481,9 +1393,7 @@ bool translate_vcmpgtps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_GT); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src2, src1, X86_FCMP_COND_GT); set_high128_xreg_to_zero(dest); } return true; @@ -1509,10 +1419,8 @@ bool translate_vcmptrueps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_TRUE); - la_xvori_b(temp, temp, 0xff); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_TRUE); + la_xvori_b(dest, dest, 0xff); set_high128_xreg_to_zero(dest); } return true; @@ -1536,9 +1444,7 @@ bool translate_vcmpeq_osps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_EQ_OS); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_EQ_OS); set_high128_xreg_to_zero(dest); } return true; @@ -1562,9 +1468,7 @@ bool translate_vcmplt_oqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_LT_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_LT_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -1588,9 +1492,7 @@ bool translate_vcmple_oqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_LE_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_LE_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -1614,9 +1516,7 @@ bool translate_vcmpunord_sps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_UNORD_S); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_UNORD_S); set_high128_xreg_to_zero(dest); } return true; @@ -1640,9 +1540,7 @@ bool translate_vcmpneq_usps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NEQ_US); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_NEQ_US); set_high128_xreg_to_zero(dest); } return true; @@ -1666,9 +1564,7 @@ bool translate_vcmpnlt_uqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_NLT_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src2, src1, X86_FCMP_COND_NLT_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -1692,9 +1588,7 @@ bool translate_vcmpnle_uqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_NLE_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src2, src1, X86_FCMP_COND_NLE_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -1718,9 +1612,7 @@ bool translate_vcmpord_sps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_ORD_S); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_ORD_S); set_high128_xreg_to_zero(dest); } return true; @@ -1744,9 +1636,7 @@ bool translate_vcmpeq_usps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_EQ_US); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_EQ_US); set_high128_xreg_to_zero(dest); } return true; @@ -1770,9 +1660,7 @@ bool translate_vcmpnge_uqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NGE_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_NGE_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -1796,9 +1684,7 @@ bool translate_vcmpngt_uqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NGT_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_NGT_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -1822,9 +1708,7 @@ bool translate_vcmpfalse_osps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_FALSE_OS); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_FALSE_OS); set_high128_xreg_to_zero(dest); } return true; @@ -1848,9 +1732,7 @@ bool translate_vcmpneq_osps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NEQ_OS); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_NEQ_OS); set_high128_xreg_to_zero(dest); } return true; @@ -1874,9 +1756,7 @@ bool translate_vcmpge_oqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_GE_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src2, src1, X86_FCMP_COND_GE_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -1900,9 +1780,7 @@ bool translate_vcmpgt_oqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_GT_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src2, src1, X86_FCMP_COND_GT_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -1928,10 +1806,8 @@ bool translate_vcmptrue_usps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_TRUE_US); - la_xvori_b(temp, temp, 0xff); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_TRUE_US); + la_xvori_b(dest, dest, 0xff); set_high128_xreg_to_zero(dest); } return true; diff --git a/target/i386/latx/translator/tr-avx-cvt.c b/target/i386/latx/translator/tr-avx-cvt.c index 4bdc082ae7..96b15dcbe0 100644 --- a/target/i386/latx/translator/tr-avx-cvt.c +++ b/target/i386/latx/translator/tr-avx-cvt.c @@ -31,10 +31,8 @@ bool translate_vcvtps2pd(IR1_INST * pir1) { } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcvtl_d_s(temp, src); - la_xvori_b(dest, temp, 0); + la_vfcvtl_d_s(dest, src); set_high128_xreg_to_zero(dest); } return true; @@ -57,11 +55,13 @@ bool translate_vcvtpd2ps(IR1_INST * pir1) { } else if (ir1_opnd_size(ir1_get_opnd(pir1, 1)) == 256) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND result = ir2_opnd_cmp(&dest, &src) ? + ra_alloc_ftemp() : dest; - la_xvpermi_q(temp, src, XVPERMI_Q_4_0(1, 1)); - la_vfcvt_s_d(temp, temp, src); - la_xvori_b(dest, temp, 0); + la_xvpermi_q(result, src, XVPERMI_Q_4_0(1, 1)); + la_vfcvt_s_d(result, result, src); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); set_high128_xreg_to_zero(dest); } else { lsassert(0); @@ -86,10 +86,8 @@ bool translate_vcvtdq2ps(IR1_INST * pir1) { } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vffint_s_w(temp, src); - la_xvori_b(dest, temp, 0); + la_vffint_s_w(dest, src); set_high128_xreg_to_zero(dest); } return true; @@ -361,20 +359,17 @@ bool translate_vcvtdq2pd(IR1_INST * pir1) { if (ir1_opnd_is_ymm(ir1_get_opnd(pir1, 0))) { IR2_OPND dest = load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); + IR2_OPND high = ra_alloc_ftemp(); - la_vffintl_d_w(temp1, src); - la_vffinth_d_w(temp2, src); - la_xvpermi_q(temp1, temp2, XVPERMI_Q_4_0(0, 2)); - la_xvori_b(dest, temp1, 0); + /* Read the overlapping source before overwriting dest's low half. */ + la_vffinth_d_w(high, src); + la_vffintl_d_w(dest, src); + la_xvpermi_q(dest, high, XVPERMI_Q_4_0(0, 2)); } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vffintl_d_w(temp, src); - la_xvori_b(dest, temp, 0); + la_vffintl_d_w(dest, src); set_high128_xreg_to_zero(dest); } set_fpu_rounding_mode(fcsr_opnd); @@ -387,7 +382,9 @@ static bool translate_vcvtpd2dq_opt(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src = load_freg256_from_ir1(opnd1); - IR2_OPND temp_f = ra_alloc_ftemp(); + bool can_write_dest = ir1_opnd_size(opnd1) == 256 && + !ir2_opnd_cmp(&dest, &src); + IR2_OPND temp_f = can_write_dest ? dest : ra_alloc_ftemp(); IR2_OPND mxcsr = ra_alloc_itemp(); IR2_OPND temp_i = ra_alloc_itemp(); int offset = lsenv_offset_of_mxcsr(lsenv); @@ -435,7 +432,8 @@ static bool translate_vcvtpd2dq_opt(IR1_INST * pir1) { la_xvpermi_d(comp_mask, comp_mask, 0x88); // la_xvshuf4i_d(comp_mask, comp_mask, 0x88); la_vbitsel_v(temp_f, temp_f, sse_invalid, comp_mask); - la_vand_v(dest, temp_f, temp_f); + if (!ir2_opnd_cmp(&temp_f, &dest)) + la_vand_v(dest, temp_f, temp_f); set_high128_xreg_to_zero(dest); } return true; @@ -584,11 +582,11 @@ bool translate_vcvtsd2ss(IR1_INST * pir1) { IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + la_fcvt_s_d(temp, src2); - la_vori_b(dest_temp, src1, 0); - la_xvinsve0_w(dest_temp, temp, 0); - la_xvori_b(dest, dest_temp, 0); + if (!ir2_opnd_cmp(&dest, &src1)) + la_vori_b(dest, src1, 0); + la_xvinsve0_w(dest, temp, 0); set_high128_xreg_to_zero(dest); set_fpu_rounding_mode(fcsr_opnd); return true; @@ -607,17 +605,20 @@ bool translate_vcvtsi2sd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_ireg_from_ir1(opnd2, UNKNOWN_EXTENSION, false); - IR2_OPND temp = ra_alloc_ftemp(); - la_movgr2fr_d(temp, src2); + IR2_OPND result = ir2_opnd_cmp(&dest, &src1) ? + ra_alloc_ftemp() : dest; + + la_movgr2fr_d(result, src2); if (ir1_opnd_size(opnd2) == 32) { - la_vffintl_d_w(temp, temp); + la_vffintl_d_w(result, result); } else if (ir1_opnd_size(opnd2) == 64) { - la_vffint_d_l(temp, temp); + la_vffint_d_l(result, result); } else { lsassert(0); } - la_vshuf4i_d(temp, src1, 0xc); - la_xvori_b(dest, temp, 0); + la_vshuf4i_d(result, src1, 0xc); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); set_high128_xreg_to_zero(dest); return true; } @@ -632,10 +633,13 @@ bool translate_vcvtss2sd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_fcvt_d_s(temp, src2); - la_vshuf4i_d(temp, src1, 0xc); - la_xvori_b(dest, temp, 0); + IR2_OPND result = ir2_opnd_cmp(&dest, &src1) ? + ra_alloc_ftemp() : dest; + + la_fcvt_d_s(result, src2); + la_vshuf4i_d(result, src1, 0xc); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); set_high128_xreg_to_zero(dest); return true; } @@ -675,7 +679,9 @@ static bool translate_vcvttpd2dq_opt(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src = load_freg256_from_ir1(opnd1); - IR2_OPND temp_f = ra_alloc_ftemp(); + bool can_write_dest = ir1_opnd_size(opnd1) == 256 && + !ir2_opnd_cmp(&dest, &src); + IR2_OPND temp_f = can_write_dest ? dest : ra_alloc_ftemp(); IR2_OPND temp_i = ra_alloc_itemp(); IR2_OPND sse_invalid = ra_alloc_ftemp(); IR2_OPND overflow = ra_alloc_ftemp(); @@ -701,7 +707,8 @@ static bool translate_vcvttpd2dq_opt(IR1_INST * pir1) { la_xvshuf4i_w(comp_mask, comp_mask, 0x88); la_xvpermi_d(comp_mask, comp_mask, 0x88); la_vbitsel_v(temp_f, temp_f, sse_invalid, comp_mask); - la_vand_v(dest, temp_f, temp_f); + if (!ir2_opnd_cmp(&temp_f, &dest)) + la_vand_v(dest, temp_f, temp_f); set_high128_xreg_to_zero(dest); } return true; diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index c2109b96da..ce52dfba82 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -34,13 +34,11 @@ bool translate_vaddpd(IR1_INST * pir1) { src2 = load_freg256_from_ir1(opnd2); la_xvfadd_d(dest, src1, src2); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); - la_vfadd_d(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vfadd_d(dest, src1, src2); set_high128_xreg_to_zero(dest); } + ra_free_temp_auto(src2); return true; } @@ -69,6 +67,7 @@ bool translate_vaddps(IR1_INST * pir1) { la_vfadd_s(dest, src1, src2); set_high128_xreg_to_zero(dest); } + ra_free_temp_auto(src2); return true; } @@ -83,11 +82,13 @@ bool translate_vaddsd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND result = ir2_opnd_cmp(&dest, &src1) ? + ra_alloc_ftemp() : dest; - la_fadd_d(temp, src1, src2); - la_vshuf4i_d(temp, src1, 0xc); - la_xvori_b(dest, temp, 0); + la_fadd_d(result, src1, src2); + la_vshuf4i_d(result, src1, 0xc); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); set_high128_xreg_to_zero(dest); return true; } @@ -136,13 +137,11 @@ bool translate_vsubpd(IR1_INST * pir1) { src2 = load_freg256_from_ir1(opnd2); la_xvfsub_d(dest, src1, src2); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); - la_vfsub_d(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vfsub_d(dest, src1, src2); set_high128_xreg_to_zero(dest); } + ra_free_temp_auto(src2); return true; } @@ -171,6 +170,7 @@ bool translate_vsubps(IR1_INST * pir1) { la_vfsub_s(dest, src1, src2); set_high128_xreg_to_zero(dest); } + ra_free_temp_auto(src2); return true; } @@ -187,10 +187,13 @@ bool translate_vsubsd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); - la_fsub_d(temp, src1, src2); - la_vshuf4i_d(temp, src1, 0xc); - la_xvori_b(dest, temp, 0); + IR2_OPND result = ir2_opnd_cmp(&dest, &src1) ? + ra_alloc_ftemp() : dest; + + la_fsub_d(result, src1, src2); + la_vshuf4i_d(result, src1, 0xc); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); set_high128_xreg_to_zero(dest); return true; } @@ -239,13 +242,11 @@ bool translate_vmulpd(IR1_INST * pir1) { src2 = load_freg256_from_ir1(opnd2); la_xvfmul_d(dest, src1, src2); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); - la_vfmul_d(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vfmul_d(dest, src1, src2); set_high128_xreg_to_zero(dest); } + ra_free_temp_auto(src2); return true; } @@ -274,6 +275,7 @@ bool translate_vmulps(IR1_INST * pir1) { la_vfmul_s(dest, src1, src2); set_high128_xreg_to_zero(dest); } + ra_free_temp_auto(src2); return true; } @@ -287,11 +289,13 @@ bool translate_vmulsd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND result = ir2_opnd_cmp(&dest, &src1) ? + ra_alloc_ftemp() : dest; - la_fmul_d(temp, src1, src2); - la_vshuf4i_d(temp, src1, 0xc); - la_xvori_b(dest, temp, 0); + la_fmul_d(result, src1, src2); + la_vshuf4i_d(result, src1, 0xc); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); set_high128_xreg_to_zero(dest); return true; } @@ -340,13 +344,11 @@ bool translate_vdivpd(IR1_INST * pir1) { src2 = load_freg256_from_ir1(opnd2); la_xvfdiv_d(dest, src1, src2); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); - la_vfdiv_d(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vfdiv_d(dest, src1, src2); set_high128_xreg_to_zero(dest); } + ra_free_temp_auto(src2); return true; } @@ -375,6 +377,7 @@ bool translate_vdivps(IR1_INST * pir1) { la_vfdiv_s(dest, src1, src2); set_high128_xreg_to_zero(dest); } + ra_free_temp_auto(src2); return true; } @@ -388,10 +391,13 @@ bool translate_vdivsd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_fdiv_d(temp, src1, src2); - la_vshuf4i_d(temp, src1, 0xc); - la_xvori_b(dest, temp, 0); + IR2_OPND result = ir2_opnd_cmp(&dest, &src1) ? + ra_alloc_ftemp() : dest; + + la_fdiv_d(result, src1, src2); + la_vshuf4i_d(result, src1, 0xc); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); set_high128_xreg_to_zero(dest); return true; } @@ -436,10 +442,8 @@ bool translate_vsqrtpd(IR1_INST * pir1) { } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfsqrt_d(temp, src); - la_xvori_b(dest, temp, 0); + la_vfsqrt_d(dest, src); set_high128_xreg_to_zero(dest); } return true; @@ -462,10 +466,8 @@ bool translate_vsqrtps(IR1_INST * pir1) { } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfsqrt_s(temp, src); - la_xvori_b(dest, temp, 0); + la_vfsqrt_s(dest, src); set_high128_xreg_to_zero(dest); } return true; @@ -618,14 +620,12 @@ bool translate_vhaddpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); la_vpickev_d(temp1, src2, src1); la_vpickod_d(temp2, src2, src1); - la_vfadd_d(temp, temp1, temp2); - la_xvori_b(dest, temp, 0); + la_vfadd_d(dest, temp1, temp2); set_high128_xreg_to_zero(dest); } return true; @@ -657,14 +657,12 @@ bool translate_vhaddps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); la_vpickev_w(temp1, src2, src1); la_vpickod_w(temp2, src2, src1); - la_vfadd_s(temp, temp1, temp2); - la_xvori_b(dest, temp, 0); + la_vfadd_s(dest, temp1, temp2); set_high128_xreg_to_zero(dest); } return true; @@ -696,14 +694,12 @@ bool translate_vhsubpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); la_vpickev_d(temp1, src2, src1); la_vpickod_d(temp2, src2, src1); - la_vfsub_d(temp, temp1, temp2); - la_xvori_b(dest, temp, 0); + la_vfsub_d(dest, temp1, temp2); set_high128_xreg_to_zero(dest); } return true; @@ -735,14 +731,12 @@ bool translate_vhsubps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); la_vpickev_w(temp1, src2, src1); la_vpickod_w(temp2, src2, src1); - la_vfsub_s(temp, temp1, temp2); - la_xvori_b(dest, temp, 0); + la_vfsub_s(dest, temp1, temp2); set_high128_xreg_to_zero(dest); } return true; @@ -3110,18 +3104,23 @@ bool translate_vfnmaddxxxpd(IR1_INST * pir1) { IR2_OPND src3_temp = ra_alloc_ftemp(); /* check if result is NaN */ - /* Unused upper lanes must not raise floating-point exceptions. */ + IR2_OPND label_over = ra_alloc_label(); if (ir1_opnd_is_xmm(opnd0)) { la_vfcmp_cond_d(mask, temp, temp, 0x8); + la_vseteqz_v(fcc0_ir2_opnd, mask); } else { la_xvfcmp_cond_d(mask, temp, temp, 0x8); + la_xvseteqz_v(fcc0_ir2_opnd, mask); } + la_bcnez(fcc0_ir2_opnd, label_over); + la_xvand_v(src1_temp, mask, dest); la_xvand_v(src2_temp, mask, src1); la_xvand_v(src3_temp, mask, src2); tr_inst(src1_temp, src2_temp, src3_temp, src1_temp); la_xvbitsel_v(temp, temp, src1_temp, mask); + la_label(label_over); la_xvori_b(dest, temp, 0); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); @@ -3184,18 +3183,23 @@ bool translate_vfnmaddxxxps(IR1_INST * pir1) { IR2_OPND src3_temp = ra_alloc_ftemp(); /* check if result is NaN */ - /* Unused upper lanes must not raise floating-point exceptions. */ + IR2_OPND label_over = ra_alloc_label(); if (ir1_opnd_is_xmm(opnd0)) { la_vfcmp_cond_s(mask, temp, temp, 0x8); + la_vseteqz_v(fcc0_ir2_opnd, mask); } else { la_xvfcmp_cond_s(mask, temp, temp, 0x8); + la_xvseteqz_v(fcc0_ir2_opnd, mask); } + la_bcnez(fcc0_ir2_opnd, label_over); + la_xvand_v(src1_temp, mask, dest); la_xvand_v(src2_temp, mask, src1); la_xvand_v(src3_temp, mask, src2); tr_inst(src1_temp, src2_temp, src3_temp, src1_temp); la_xvbitsel_v(temp, temp, src1_temp, mask); + la_label(label_over); la_xvori_b(dest, temp, 0); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); @@ -3353,18 +3357,23 @@ bool translate_vfnmsubxxxpd(IR1_INST * pir1) { IR2_OPND src3_temp = ra_alloc_ftemp(); /* check if result is NaN */ - /* Unused upper lanes must not raise floating-point exceptions. */ + IR2_OPND label_over = ra_alloc_label(); if (ir1_opnd_is_xmm(opnd0)) { la_vfcmp_cond_d(mask, temp, temp, 0x8); + la_vseteqz_v(fcc0_ir2_opnd, mask); } else { la_xvfcmp_cond_d(mask, temp, temp, 0x8); + la_xvseteqz_v(fcc0_ir2_opnd, mask); } + la_bcnez(fcc0_ir2_opnd, label_over); + la_xvand_v(src1_temp, mask, dest); la_xvand_v(src2_temp, mask, src1); la_xvand_v(src3_temp, mask, src2); tr_inst(src1_temp, src2_temp, src3_temp, src1_temp); la_xvbitsel_v(temp, temp, src1_temp, mask); + la_label(label_over); la_xvori_b(dest, temp, 0); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); @@ -3424,18 +3433,23 @@ bool translate_vfnmsubxxxps(IR1_INST * pir1) { IR2_OPND src3_temp = ra_alloc_ftemp(); /* check if result is NaN */ - /* Unused upper lanes must not raise floating-point exceptions. */ + IR2_OPND label_over = ra_alloc_label(); if (ir1_opnd_is_xmm(opnd0)) { la_vfcmp_cond_s(mask, temp, temp, 0x8); + la_vseteqz_v(fcc0_ir2_opnd, mask); } else { la_xvfcmp_cond_s(mask, temp, temp, 0x8); + la_xvseteqz_v(fcc0_ir2_opnd, mask); } + la_bcnez(fcc0_ir2_opnd, label_over); + la_xvand_v(src1_temp, mask, dest); la_xvand_v(src2_temp, mask, src1); la_xvand_v(src3_temp, mask, src2); tr_inst(src1_temp, src2_temp, src3_temp, src1_temp); la_xvbitsel_v(temp, temp, src1_temp, mask); + la_label(label_over); la_xvori_b(dest, temp, 0); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index 02e1e2656c..938a04ff17 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -1,4 +1,13 @@ if 'x86_64-linux-user' in target_dirs + foreach cvt_option : ['0', '1'] + latx_integration_tests += [{ + 'name': 'test-avx-conversion-alias-' + cvt_option, + 'runner': find_program('../../test-avx-conversion-alias.py'), + 'args': [emulators['latx-x86_64'], files('../../vcvtpd-register-alias.asm')], + 'env': {'LATX_SOFTFPU': '0', 'LATX_CVT_OPT': cvt_option}, + 'timeout': 120, + }] + endforeach if 'CONFIG_LATX_AVX_OPT' in config_host latx_integration_tests += [{ 'name': 'test-f16c-memory-boundary', @@ -78,6 +87,7 @@ if 'x86_64-linux-user' in target_dirs files('../../avx-ymm-writeback.S'), files('../../vpshufd-distinct-regs.S'), files('../../vroundps-imm0b.S'), + files('../../vcvtdq2pd-ymm-alias.asm'), files('../../avx-regression-extract-alias.S'), files('../../avx-regression-narrow-state.S'), files('../../avx-regression-fma-alias.S'), diff --git a/tests/integration/test-avx-conversion-alias.py b/tests/integration/test-avx-conversion-alias.py new file mode 100755 index 0000000000..0dee8ccfe1 --- /dev/null +++ b/tests/integration/test-avx-conversion-alias.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 LAT Project Authors +# SPDX-License-Identifier: GPL-2.0-only +"""Reject unsupported-register diagnostics as well as wrong AVX results.""" +from pathlib import Path +import subprocess +import sys + +runner = Path(__file__).with_name("test-avx-regressions.py") +result = subprocess.run([sys.executable, str(runner), *sys.argv[1:]], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True) +print(result.stdout, end="") +if "reg no match" in result.stdout: + print("FAIL: conversion used a register check that does not support its operands") + sys.exit(1) +sys.exit(result.returncode) diff --git a/tests/integration/vcvtdq2pd-ymm-alias.asm b/tests/integration/vcvtdq2pd-ymm-alias.asm new file mode 100644 index 0000000000..04317cfbdb --- /dev/null +++ b/tests/integration/vcvtdq2pd-ymm-alias.asm @@ -0,0 +1,64 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +section .text + +%macro CHECK_RESULT 1 + vmovdqu [actual], ymm0 + mov rax, [actual] + cmp rax, [%1] + jne fail + mov rax, [actual + 8] + cmp rax, [%1 + 8] + jne fail + mov rax, [actual + 16] + cmp rax, [%1 + 16] + jne fail + mov rax, [actual + 24] + cmp rax, [%1 + 24] + jne fail +%endmacro + +_start: + vmovdqu xmm1, [input] + vpxor ymm0, ymm0, ymm0 + vcvtdq2pd ymm0, xmm1 + CHECK_RESULT expected + + vmovdqu ymm1, [alias_input] + vcvtdq2pd ymm1, xmm1 + vmovdqu ymm0, ymm1 + CHECK_RESULT expected + + vpxor ymm0, ymm0, ymm0 + vcvtdq2pd ymm0, [input] + CHECK_RESULT expected + + vmovdqu ymm2, [alias_input] + vcvtdq2pd ymm2, [input] + vmovdqu ymm0, ymm2 + CHECK_RESULT expected + + xor edi, edi +fail: + mov eax, 60 + syscall + +section .rodata align=32 +input: + dd 0, 0x80000000, 0x7fffffff, -1 + dd 1, 2, 3, 4 +alias_input: + dd 0, 0x80000000, 0x7fffffff, -1 + dq 0xdeadbeefdeadbeef, 0xdeadbeefdeadbeef +expected: + dq 0x0000000000000000, 0xc1e0000000000000 + dq 0x41dfffffffc00000, 0xbff0000000000000 + +section .bss align=32 +actual: resb 32 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vcvtpd-register-alias.asm b/tests/integration/vcvtpd-register-alias.asm new file mode 100644 index 0000000000..947bbb95a1 --- /dev/null +++ b/tests/integration/vcvtpd-register-alias.asm @@ -0,0 +1,81 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +BITS 64 +default rel +global _start +section .text + +%macro CONVERT 4 + vmovdqu ymm0, [poison] + vmovdqu ymm1, [input] +%if %4 = 0 + vmovdqu ymm0, [input] +%if %3 = 256 + %1 xmm0, ymm0 +%else + %1 xmm0, xmm0 +%endif +%elif %4 = 1 +%if %3 = 256 + %1 xmm0, ymm1 +%else + %1 xmm0, xmm1 +%endif +%else +%if %3 = 256 + %1 xmm0, yword [input] +%else + %1 xmm0, oword [input] +%endif +%endif + vmovdqu [output], ymm0 +%if %2 + mov rax, 0x400000003f800000 +%else + mov rax, 0x0000000200000001 +%endif + cmp [output], rax + jne fail +%if %3 = 128 + xor eax, eax +%elif %2 + mov rax, 0x4080000040400000 +%else + mov rax, 0x0000000400000003 +%endif + cmp [output + 8], rax + jne fail + cmp qword [output + 16], 0 + jne fail + cmp qword [output + 24], 0 + jne fail +%endmacro + +_start: +; Repeat translations so a broken release build flushes its buffered diagnostic. +%rep 32 +%assign kind 0 +%rep 3 + CONVERT vcvtpd2ps, 1, 128, kind + CONVERT vcvtpd2ps, 1, 256, kind + CONVERT vcvtpd2dq, 0, 128, kind + CONVERT vcvtpd2dq, 0, 256, kind + CONVERT vcvttpd2dq, 0, 128, kind + CONVERT vcvttpd2dq, 0, 256, kind +%assign kind kind + 1 +%endrep +%endrep + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +input: dq 1.0, 2.0, 3.0, 4.0 +poison: times 4 dq -1 +section .bss align=32 +output: resb 32 +section .note.GNU-stack noalloc noexec nowrite progbits From 4306df8388c442993cc9be097338ed2819ac7d44 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 31 Aug 2026 17:57:54 +0800 Subject: [PATCH 02/33] latx: reduce copies in AVX moves and shuffles Write move, broadcast, insert, extract and shuffle results directly to the destination when the source lanes are no longer needed. Keep the XMM VSHUFPD single-shuffle path when both sources are the same register. (cherry picked from commit c0084ea3335a0e7a191f5841bdbd418d69df4841) Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx-mov.c | 92 ++++++------ target/i386/latx/translator/tr-avx.c | 183 +++++++++++------------ 2 files changed, 133 insertions(+), 142 deletions(-) diff --git a/target/i386/latx/translator/tr-avx-mov.c b/target/i386/latx/translator/tr-avx-mov.c index baaa387923..886ca92633 100644 --- a/target/i386/latx/translator/tr-avx-mov.c +++ b/target/i386/latx/translator/tr-avx-mov.c @@ -75,8 +75,9 @@ bool translate_vmovaps(IR1_INST * pir1) { store_freg256_to_ir1_mem(ra_alloc_xmm(ir1_opnd_base_reg_num(src)), dest); } else if (ir1_opnd_is_ymm(dest) && ir1_opnd_is_ymm(src)) { - la_xvori_b(ra_alloc_xmm(ir1_opnd_base_reg_num(dest)), - ra_alloc_xmm(ir1_opnd_base_reg_num(src)), 0); + if (!ir1_opnd_is_same_reg(dest, src)) + la_xvori_b(ra_alloc_xmm(ir1_opnd_base_reg_num(dest)), + ra_alloc_xmm(ir1_opnd_base_reg_num(src)), 0); } else if (ir1_opnd_is_xmm(dest) && ir1_opnd_is_mem(src)) { IR2_OPND dest_reg = ra_alloc_xmm(ir1_opnd_base_reg_num(dest)); @@ -88,7 +89,8 @@ bool translate_vmovaps(IR1_INST * pir1) { } else if (ir1_opnd_is_xmm(dest) && ir1_opnd_is_xmm(src)) { IR2_OPND dest_reg = ra_alloc_xmm(ir1_opnd_base_reg_num(dest)); - la_vori_b(dest_reg, ra_alloc_xmm(ir1_opnd_base_reg_num(src)), 0); + if (!ir1_opnd_is_same_reg(dest, src)) + la_vori_b(dest_reg, ra_alloc_xmm(ir1_opnd_base_reg_num(src)), 0); set_high128_xreg_to_zero(dest_reg); } else { #ifdef CONFIG_LATX_TS @@ -113,15 +115,14 @@ bool translate_vmovmskps(IR1_INST * pir1) { ra_alloc_xmm(ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1)))); la_movfr2gr_d(dest, temp); } else { - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); + IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND high = ra_alloc_itemp(); - la_xvmskltz_w(temp1, + la_xvmskltz_w(temp, ra_alloc_xmm(ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1)))); - la_xvpermi_q(temp2, temp1, VEXTRINS_IMM_4_0(1, 1)); - la_vslli_b(temp2, temp2, 4); - la_vor_v(temp1, temp1, temp2); - la_movfr2gr_d(dest, temp1); + la_xvpickve2gr_du(dest, temp, 0); + la_xvpickve2gr_du(high, temp, 2); + la_bstrins_d(dest, high, 7, 4); } return true; } @@ -140,15 +141,14 @@ bool translate_vmovmskpd(IR1_INST * pir1) { ra_alloc_xmm(ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1)))); la_movfr2gr_d(dest, temp); } else { - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); + IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND high = ra_alloc_itemp(); - la_xvmskltz_d(temp1, + la_xvmskltz_d(temp, ra_alloc_xmm(ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1)))); - la_xvpermi_q(temp2, temp1, VEXTRINS_IMM_4_0(1, 1)); - la_vslli_b(temp2, temp2, 2); - la_vor_v(temp1, temp1, temp2); - la_movfr2gr_d(dest, temp1); + la_xvpickve2gr_du(dest, temp, 0); + la_xvpickve2gr_du(high, temp, 2); + la_bstrins_d(dest, high, 3, 2); } return true; } @@ -207,6 +207,7 @@ bool translate_vmovshdup(IR1_INST * pir1) { } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); + la_vpackod_w(dest, src, src); set_high128_xreg_to_zero(dest); } @@ -227,10 +228,8 @@ bool translate_vmovsldup(IR1_INST * pir1) { } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvpackev_w(temp, src, src); - la_xvori_b(dest, temp, 0); + la_xvpackev_w(dest, src, src); set_high128_xreg_to_zero(dest); } return true; @@ -251,10 +250,8 @@ bool translate_vmovddup(IR1_INST * pir1) { } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvrepl128vei_d(temp, src, 0); - la_xvori_b(dest, temp, 0); + la_xvrepl128vei_d(dest, src, 0); set_high128_xreg_to_zero(dest); } return true; @@ -310,11 +307,15 @@ bool translate_vmovss(IR1_INST *pir1) IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); if (ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_xmm(opnd1)) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - la_xvori_b(temp, src1, 0); - la_xvinsve0_w(temp, src2, 0); - la_xvori_b(dest, temp, 0); + IR2_OPND result = ir2_opnd_cmp(&dest, &src2) ? + ra_alloc_ftemp() : dest; + + if (!ir2_opnd_cmp(&result, &src1)) + la_vori_b(result, src1, 0); + la_xvinsve0_w(result, src2, 0); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); set_high128_xreg_to_zero(dest); } else if (ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_mem(opnd1)) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); @@ -346,7 +347,6 @@ bool translate_vmovd(IR1_INST *pir1) if (ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_mem(opnd1)) { IR2_OPND src = load_freg_from_ir1_1(opnd1, false, IS_INTEGER); IR2_OPND dest = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd0)); - la_xvandi_b(dest, dest, 0); la_xvpickve_w(dest, src, 0); set_high128_xreg_to_zero(dest); } else if (ir1_opnd_is_mem(opnd0) && ir1_opnd_is_xmm(opnd1)) { @@ -536,8 +536,12 @@ bool translate_vmovlpd(IR1_INST * pir1) { IR2_OPND temp_src2 = load_freg128_from_ir1(src2); IR2_OPND dest_temp = ra_alloc_xmm(ir1_opnd_base_reg_num(dest)); IR2_OPND src_temp = ra_alloc_xmm(ir1_opnd_base_reg_num(src)); - la_vreplvei_d(dest_temp, src_temp, 1); - la_xvinsve0_d(dest_temp, temp_src2, 0); + if (ir1_opnd_is_same_reg(dest, src)) { + la_vshuf4i_d(dest_temp, temp_src2, 0x6); + } else { + la_vreplvei_d(dest_temp, src_temp, 1); + la_xvinsve0_d(dest_temp, temp_src2, 0); + } set_high128_xreg_to_zero(dest_temp); return true; } else if (ir1_opnd_is_mem(dest) && ir1_opnd_is_xmm(src)) { @@ -615,20 +619,7 @@ bool translate_vmovntdqa(IR1_INST * pir1) { if (!option_enable_lasx) { return translate_vmovntdqa_lsx(pir1); } - IR1_OPND * dest = ir1_get_opnd(pir1, 0); - IR1_OPND * src = ir1_get_opnd(pir1, 1); - if (ir1_opnd_is_xmm(dest) && ir1_opnd_is_mem(src)) { - IR2_OPND temp = load_freg128_from_ir1(src); - IR2_OPND temp_dest = ra_alloc_xmm(ir1_opnd_base_reg_num(dest)); - la_vand_v(temp_dest, temp, temp); - set_high128_xreg_to_zero(temp_dest); - - } else if (ir1_opnd_is_ymm(dest) && ir1_opnd_is_mem(src)) { - IR2_OPND temp = load_freg256_from_ir1(src); - la_xvand_v(ra_alloc_xmm(ir1_opnd_base_reg_num(dest)), - temp, temp); - } - return true; + return translate_vmovaps(pir1); } bool translate_vmovsd(IR1_INST * pir1) { @@ -644,11 +635,14 @@ bool translate_vmovsd(IR1_INST * pir1) { IR2_OPND temp_src1 = load_freg128_from_ir1(src); IR2_OPND temp_src2 = load_freg128_from_ir1(src2); - IR2_OPND temp = ra_alloc_ftemp(); - - la_xvori_b(temp, temp_src1, 0x0); - la_vshuf4i_d(temp, temp_src2, 0b00000110); - la_xvori_b(temp_dest, temp, 0x0); + IR2_OPND result = ir2_opnd_cmp(&temp_dest, &temp_src2) ? + ra_alloc_ftemp() : temp_dest; + + if (!ir2_opnd_cmp(&result, &temp_src1)) + la_vori_b(result, temp_src1, 0x0); + la_vshuf4i_d(result, temp_src2, 0b00000110); + if (!ir2_opnd_cmp(&result, &temp_dest)) + la_xvori_b(temp_dest, result, 0x0); set_high128_xreg_to_zero(temp_dest); return true; } diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index ce52dfba82..17a10f2e6f 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -1332,15 +1332,15 @@ bool translate_vextractps(IR1_INST * pir1) { IR1_OPND * opnd2 = ir1_get_opnd(pir1, 2); lsassert(ir1_opnd_is_gpr(opnd0) || ir1_opnd_is_mem(opnd0)); IR2_OPND src = load_freg128_from_ir1(opnd1); - IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND value = ra_alloc_itemp(); uint8_t imm = ir1_opnd_uimm(opnd2) & 0x3; - la_xvpickve_w(temp, src, imm); + la_xvpickve2gr_w(value, src, imm); if (ir1_opnd_is_gpr(opnd0)) { - IR2_OPND dest = ra_alloc_gpr(ir1_opnd_base_reg_num(opnd0)); - la_movfr2gr_d(dest, temp); + store_ireg_to_ir1(value, opnd0, false); } else { - store_freg_to_ir1(temp, opnd0, false, false); + store_u32_to_ir1_mem_exact(value, opnd0); } + ra_free_temp(value); return true; } @@ -1501,7 +1501,8 @@ bool translate_vshufpd(IR1_INST * pir1) { IR2_OPND src1 = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp_l = ra_alloc_ftemp(); - IR2_OPND temp_h = ra_alloc_ftemp(); + IR2_OPND result = ir2_opnd_cmp(&dest, &src2) ? + ra_alloc_ftemp() : dest; uint8_t imm8 = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)) & 0xf; uint8_t l = imm8 & 0x3; uint8_t h = imm8 >> 2; @@ -1529,27 +1530,27 @@ bool translate_vshufpd(IR1_INST * pir1) { lsassert(0); } la_xvori_b(temp_l, src1, 0); - la_xvori_b(temp_h, src1, 0); la_xvshuf4i_d(temp_l, src2, l); - la_xvshuf4i_d(temp_h, src2, h); - la_xvpermi_q(temp_h, temp_l, VEXTRINS_IMM_4_0(3, 0)); - la_xvori_b(dest, temp_h, 0); + if (!ir2_opnd_cmp(&result, &src1)) + la_xvori_b(result, src1, 0); + la_xvshuf4i_d(result, src2, h); + la_xvpermi_q(result, temp_l, VEXTRINS_IMM_4_0(3, 0)); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - bool same_sources = ir1_opnd_is_xmm(ir1_get_opnd(pir1, 2)) && - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1)) == - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 2)); - bool dest_is_src1 = ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) == - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1)); - bool dest_is_src2 = ir1_opnd_is_xmm(ir1_get_opnd(pir1, 2)) && - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) == - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 2)); - bool write_dest = dest_is_src1 || !dest_is_src2; - IR2_OPND temp = write_dest ? dest : ra_alloc_ftemp(); + IR2_OPND result = dest; uint8_t imm8 = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)); imm8 = imm8 & 3; + if (ir2_opnd_cmp(&src1, &src2)) { + static const uint8_t permi[4] = {0x44, 0x4e, 0xe4, 0xee}; + + la_vshuf4i_w(dest, src1, permi[imm8]); + set_high128_xreg_to_zero(dest); + return true; + } if (imm8 == 0) { imm8 = 0x8; } else if (imm8 == 1) { @@ -1561,21 +1562,13 @@ bool translate_vshufpd(IR1_INST * pir1) { } else { lsassert(0); } - if (same_sources) { - static const uint8_t permi[4] = {0x44, 0x4e, 0xe4, 0xee}; - - la_vshuf4i_w(dest, src1, permi[ - ir1_opnd_uimm(ir1_get_opnd(pir1, 3)) & 3]); - set_high128_xreg_to_zero(dest); - return true; - } - if (!dest_is_src1) { - la_vori_b(temp, src1, 0); - } - la_vshuf4i_d(temp, src2, imm8); - if (!write_dest) { - la_vori_b(dest, temp, 0); - } + if (ir2_opnd_cmp(&dest, &src2) && !ir2_opnd_cmp(&dest, &src1)) + result = ra_alloc_ftemp(); + if (!ir2_opnd_cmp(&result, &src1)) + la_vori_b(result, src1, 0); + la_vshuf4i_d(result, src2, imm8); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); set_high128_xreg_to_zero(dest); } return true; @@ -1623,8 +1616,7 @@ bool translate_vshufps(IR1_INST * pir1) { IR2_OPND temp2 = ra_alloc_ftemp(); la_vshuf4i_w(temp1, src1, imm8); la_vshuf4i_w(temp2, src2, imm8 >> 4); - la_vpickev_d(temp1, temp2, temp1); - la_xvori_b(dest, temp1, 0); + la_vpickev_d(dest, temp2, temp1); set_high128_xreg_to_zero(dest); } return true; @@ -1644,20 +1636,14 @@ bool translate_vunpckhpd(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvori_b(temp, src1, 0); - la_xvshuf4i_d(temp, src2, 0xd); - la_xvori_b(dest, temp, 0); + la_xvilvh_d(dest, src2, src1); } else { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vori_b(temp, src1, 0); - la_vshuf4i_d(temp, src2, 0xd); - la_xvori_b(dest, temp, 0); + la_xvilvh_d(dest, src2, src1); set_high128_xreg_to_zero(dest); } return true; @@ -1683,10 +1669,8 @@ bool translate_vunpckhps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vilvh_w(temp, src2, src1); - la_xvori_b(dest, temp, 0); + la_vilvh_w(dest, src2, src1); set_high128_xreg_to_zero(dest); } return true; @@ -1706,20 +1690,14 @@ bool translate_vunpcklpd(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvori_b(temp, src1, 0); - la_xvshuf4i_d(temp, src2, 0x8); - la_xvori_b(dest, temp, 0); + la_xvilvl_d(dest, src2, src1); } else { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vori_b(temp, src1, 0); - la_vshuf4i_d(temp, src2, 0x8); - la_xvori_b(dest, temp, 0); + la_xvilvl_d(dest, src2, src1); set_high128_xreg_to_zero(dest); } return true; @@ -1745,10 +1723,8 @@ bool translate_vunpcklps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vilvl_w(temp, src2, src1); - la_xvori_b(dest, temp, 0); + la_vilvl_w(dest, src2, src1); set_high128_xreg_to_zero(dest); } return true; @@ -1765,7 +1741,6 @@ bool translate_vpabsx(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src = load_freg256_from_ir1(opnd1); IR2_OPND vzero = ra_alloc_ftemp(); - IR2_OPND temp = ra_alloc_ftemp(); IR2_INST * ( * tr_inst_128)(IR2_OPND, IR2_OPND, IR2_OPND); IR2_INST * ( * tr_inst_256)(IR2_OPND, IR2_OPND, IR2_OPND); switch (ir1_opcode(pir1)) { @@ -1790,10 +1765,9 @@ bool translate_vpabsx(IR1_INST * pir1) { la_vreplgr2vr_d(vzero, zero_ir2_opnd); if (ir1_opnd_is_ymm(opnd0)) - tr_inst_256(temp, src, vzero); + tr_inst_256(dest, src, vzero); else - tr_inst_128(temp, src, vzero); - la_xvori_b(dest, temp, 0); + tr_inst_128(dest, src, vzero); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); @@ -2165,18 +2139,23 @@ bool translate_vperm2i128(IR1_INST * pir1) { } IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvori_b(temp, src2, 0); - la_xvpermi_q(temp, src1, imm); + IR2_OPND result = dest; + + if (ir2_opnd_cmp(&dest, &src1) && !ir2_opnd_cmp(&src1, &src2)) + result = ra_alloc_ftemp(); + if (!ir2_opnd_cmp(&result, &src2)) + la_xvori_b(result, src2, 0); + la_xvpermi_q(result, src1, imm); if (zero_l) { - la_xvinsgr2vr_d(temp, zero_ir2_opnd, 0); - la_xvinsgr2vr_d(temp, zero_ir2_opnd, 1); + la_xvinsgr2vr_d(result, zero_ir2_opnd, 0); + la_xvinsgr2vr_d(result, zero_ir2_opnd, 1); } if (zero_h) { - la_xvinsgr2vr_d(temp, zero_ir2_opnd, 2); - la_xvinsgr2vr_d(temp, zero_ir2_opnd, 3); + la_xvinsgr2vr_d(result, zero_ir2_opnd, 2); + la_xvinsgr2vr_d(result, zero_ir2_opnd, 3); } - la_xvori_b(dest, temp, 0); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); return true; } @@ -4382,7 +4361,8 @@ bool translate_vpinsrx(IR1_INST * pir1) { } - la_vori_b(dest, src1, 0x0); + if (!ir2_opnd_cmp(&dest, &src1)) + la_vori_b(dest, src1, 0x0); tr_inst(dest, src2, imm); set_high128_xreg_to_zero(dest); return true; @@ -4561,10 +4541,15 @@ bool translate_vpermilps(IR1_INST * pir1) { } else { IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); - la_vandi_b(temp, src2, 0b00000011); - la_vshuf_w(temp, src2, src1); - la_vori_b(dest, temp, 0x0); + IR2_OPND result = dest; + + if (ir2_opnd_cmp(&dest, &src1) || + ir2_opnd_cmp(&dest, &src2)) + result = ra_alloc_ftemp(); + la_vandi_b(result, src2, 0b00000011); + la_vshuf_w(result, src2, src1); + if (!ir2_opnd_cmp(&result, &dest)) + la_vori_b(dest, result, 0); } set_high128_xreg_to_zero(dest); } else { @@ -4576,11 +4561,15 @@ bool translate_vpermilps(IR1_INST * pir1) { } else { IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvandi_b(temp, src2, 0b00000011); - la_xvshuf_w(temp, src2, src1); - la_xvori_b(dest, temp, 0x0); - + IR2_OPND result = dest; + + if (ir2_opnd_cmp(&dest, &src1) || + ir2_opnd_cmp(&dest, &src2)) + result = ra_alloc_ftemp(); + la_xvandi_b(result, src2, 0b00000011); + la_xvshuf_w(result, src2, src1); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); } } @@ -4606,12 +4595,16 @@ bool translate_vpermilpd(IR1_INST * pir1) { } else { IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); - la_vsrli_d(temp, src2, 1); - la_vandi_b(temp, temp, 0b00000001); - la_vshuf_d(temp, src2, src1); - la_vori_b(dest, temp, 0x0); - + IR2_OPND result = dest; + + if (ir2_opnd_cmp(&dest, &src1) || + ir2_opnd_cmp(&dest, &src2)) + result = ra_alloc_ftemp(); + la_vsrli_d(result, src2, 1); + la_vandi_b(result, result, 0b00000001); + la_vshuf_d(result, src2, src1); + if (!ir2_opnd_cmp(&result, &dest)) + la_vori_b(dest, result, 0); } set_high128_xreg_to_zero(dest); } else { @@ -4624,12 +4617,16 @@ bool translate_vpermilpd(IR1_INST * pir1) { } else { IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvsrli_d(temp, src2, 1); - la_xvandi_b(temp, temp, 0b00000001); - la_xvshuf_d(temp, src2, src1); - la_xvori_b(dest, temp, 0x0); - + IR2_OPND result = dest; + + if (ir2_opnd_cmp(&dest, &src1) || + ir2_opnd_cmp(&dest, &src2)) + result = ra_alloc_ftemp(); + la_xvsrli_d(result, src2, 1); + la_xvandi_b(result, result, 0b00000001); + la_xvshuf_d(result, src2, src1); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); } } From 52b19c9a92121971fc328a5b1f40ecc2e4b20623 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 31 Aug 2026 17:57:54 +0800 Subject: [PATCH 03/33] latx: reduce copies and mask construction in AVX blends Use copies and lane inserts for constant VBLENDPD and VBLENDPS masks. Write XMM VBLENDVPD results directly to the destination. For XMM VBLENDPD with dest equal to src2, retain the selected dest lanes and insert only the lanes from src1. This removes the temporary result and its copies. (cherry picked from commit ef2925a0d248a339c0781fca4a2c59a28b0d2c09) Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx.c | 164 ++++++++++++------ .../integration/registrations/avx/meson.build | 1 + tests/integration/vblendpd-xmm-alias.asm | 109 ++++++++++++ 3 files changed, 224 insertions(+), 50 deletions(-) create mode 100644 tests/integration/vblendpd-xmm-alias.asm diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 17a10f2e6f..4eb5778836 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -1096,8 +1096,7 @@ bool translate_vblendvpd(IR1_INST * pir1) { IR2_OPND src3 = load_freg128_from_ir1(opnd3); IR2_OPND temp = ra_alloc_ftemp(); la_vslti_d(temp, src3, 0); - la_vbitsel_v(temp, src1, src2, temp); - la_xvori_b(dest, temp, 0); + la_vbitsel_v(dest, src1, src2, temp); set_high128_xreg_to_zero(dest); } return true; @@ -1118,24 +1117,75 @@ bool translate_vblendpd(IR1_INST * pir1) { IR2_OPND src1 = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); uint8 imm = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvori_b(temp, src1, 0); - if (imm & 0x1) - la_vextrins_d(temp, src2, VEXTRINS_IMM_4_0(0, 0)); - if (imm & 0x2) - la_vextrins_d(temp, src2, VEXTRINS_IMM_4_0(1, 1)); - if (ir1_opnd_is_ymm(ir1_get_opnd(pir1, 0))) { - IR2_OPND temp2 = ra_alloc_ftemp(); - la_xvori_b(temp2, src1, 0); - if (imm & 0x4) - la_xvextrins_d(temp2, src2, VEXTRINS_IMM_4_0(0, 0)); - if (imm & 0x8) - la_xvextrins_d(temp2, src2, VEXTRINS_IMM_4_0(1, 1)); - la_xvpermi_q(temp, temp2, VEXTRINS_IMM_4_0(1, 2)); + uint8 lane_mask = ir1_opnd_is_ymm(ir1_get_opnd(pir1, 0)) ? + 0xf : 0x3; + + if ((imm & lane_mask) == 0) { + if (!ir2_opnd_cmp(&dest, &src1)) + la_xvori_b(dest, src1, 0); + if (ir1_opnd_is_xmm(ir1_get_opnd(pir1, 0))) + set_high128_xreg_to_zero(dest); + return true; } - la_xvori_b(dest, temp, 0); - if (ir1_opnd_is_xmm(ir1_get_opnd(pir1, 0))) + if ((imm & lane_mask) == lane_mask) { + if (!ir2_opnd_cmp(&dest, &src2)) + la_xvori_b(dest, src2, 0); + if (ir1_opnd_is_xmm(ir1_get_opnd(pir1, 0))) + set_high128_xreg_to_zero(dest); + return true; + } + + if (ir1_opnd_is_xmm(ir1_get_opnd(pir1, 0))) { + if (ir2_opnd_cmp(&dest, &src1) && ir2_opnd_cmp(&dest, &src2)) { + set_high128_xreg_to_zero(dest); + return true; + } + if (ir2_opnd_cmp(&dest, &src2) && !ir2_opnd_cmp(&dest, &src1)) { + if (!(imm & 0x1)) + la_vextrins_d(dest, src1, VEXTRINS_IMM_4_0(0, 0)); + if (!(imm & 0x2)) + la_vextrins_d(dest, src1, VEXTRINS_IMM_4_0(1, 1)); + set_high128_xreg_to_zero(dest); + return true; + } + IR2_OPND result = ir2_opnd_cmp(&dest, &src2) ? + ra_alloc_ftemp() : dest; + + if (!ir2_opnd_cmp(&result, &src1)) + la_vori_b(result, src1, 0); + if (imm & 0x1) + la_vextrins_d(result, src2, VEXTRINS_IMM_4_0(0, 0)); + if (imm & 0x2) + la_vextrins_d(result, src2, VEXTRINS_IMM_4_0(1, 1)); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); set_high128_xreg_to_zero(dest); + return true; + } + + bool use_src2_base = __builtin_popcount(imm & lane_mask) > 2; + uint8 insert_mask = use_src2_base ? (~imm & lane_mask) : + (imm & lane_mask); + IR2_OPND base = use_src2_base ? src2 : src1; + IR2_OPND insert_src = use_src2_base ? src1 : src2; + IR2_OPND result = (!ir2_opnd_cmp(&dest, &src1) && + !ir2_opnd_cmp(&dest, &src2)) ? + dest : ra_alloc_ftemp(); + + la_xvori_b(result, base, 0); + if (insert_mask & 0x1) + la_vextrins_d(result, insert_src, VEXTRINS_IMM_4_0(0, 0)); + if (insert_mask & 0x2) + la_vextrins_d(result, insert_src, VEXTRINS_IMM_4_0(1, 1)); + IR2_OPND high = ra_alloc_ftemp(); + la_xvori_b(high, base, 0); + if (insert_mask & 0x4) + la_xvextrins_d(high, insert_src, VEXTRINS_IMM_4_0(0, 0)); + if (insert_mask & 0x8) + la_xvextrins_d(high, insert_src, VEXTRINS_IMM_4_0(1, 1)); + la_xvpermi_q(result, high, VEXTRINS_IMM_4_0(1, 2)); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); return true; } @@ -1144,48 +1194,62 @@ bool translate_vblendps(IR1_INST * pir1) { return translate_vblendps_lsx(pir1); } + IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); + IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); + IR1_OPND * opnd2 = ir1_get_opnd(pir1, 2); lsassert(ir1_opnd_num(pir1) == 4 && ir1_opnd_is_imm(ir1_get_opnd(pir1, 3))); - lsassert((ir1_opnd_is_xmm(ir1_get_opnd(pir1, 0)) && - ir1_opnd_is_xmm(ir1_get_opnd(pir1, 1))) || - (ir1_opnd_is_ymm(ir1_get_opnd(pir1, 0)) && - ir1_opnd_is_ymm(ir1_get_opnd(pir1, 1)))); - IR2_OPND dest = load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); - IR2_OPND src1 = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); + lsassert((ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_xmm(opnd1)) || + (ir1_opnd_is_ymm(opnd0) && ir1_opnd_is_ymm(opnd1))); + IR2_OPND dest = load_freg256_from_ir1(opnd0); + IR2_OPND src1 = load_freg256_from_ir1(opnd1); + IR2_OPND src2 = load_freg256_from_ir1(opnd2); uint8 imm = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)); + int lane_count = ir1_opnd_is_ymm(opnd0) ? 8 : 4; + uint8_t all_lanes_mask = (1u << lane_count) - 1; + uint8_t lane_mask = imm & all_lanes_mask; + bool dest_is_src2 = ir1_opnd_is_same_reg(opnd0, opnd2); + bool dest_is_src1 = ir1_opnd_is_same_reg(opnd0, opnd1); + uint8_t insert_mask = dest_is_src2 ? ~lane_mask : lane_mask; + int insert_count = __builtin_popcount(insert_mask & + all_lanes_mask); - if (ir1_opnd_is_xmm(ir1_get_opnd(pir1, 0)) && - ir1_opnd_is_xmm(ir1_get_opnd(pir1, 2)) && - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 2))) { - if (ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) { - la_vori_b(dest, src1, 0); + /* Both sources are loaded before these copies, including a memory src2. */ + if (lane_mask == 0) { + if (!dest_is_src1) + la_xvori_b(dest, src1, 0); + } else if (lane_mask == all_lanes_mask) { + if (!dest_is_src2) + la_xvori_b(dest, src2, 0); + } else if (ir1_opnd_is_xmm(opnd0) && + insert_count + (!dest_is_src2 && !dest_is_src1) <= 3) { + IR2_OPND insert_src = dest_is_src2 ? src1 : src2; + + if (!dest_is_src2 && !dest_is_src1) { + la_xvori_b(dest, src1, 0); } - for (int i = 0; i < 4; ++i) { - if (imm & (1 << i)) { - la_vextrins_w(dest, src2, VEXTRINS_IMM_4_0(i, i)); + for (int i = 0; i < lane_count; i++) { + if (insert_mask & (1u << i)) { + la_vextrins_w(dest, insert_src, + VEXTRINS_IMM_4_0(i, i)); } } - set_high128_xreg_to_zero(dest); - return true; - } - - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND rmask = ra_alloc_itemp(); + } else { + IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND rmask = ra_alloc_itemp(); + uint64_t mask = 0; - uint64_t mask = 0; - for (int i = 0; i < 8; i++) { - if (imm & (1 << i)) { - mask |= (0xFFULL) << (i * 8); + for (int i = 0; i < 8; i++) { + if (imm & (1 << i)) { + mask |= (0xFFULL) << (i * 8); + } } - } - li_d(rmask, mask); - la_movgr2fr_d(temp, rmask); - la_vext2xv_w_b(temp, temp); - la_xvbitsel_v(dest, src1, src2, temp); + li_d(rmask, mask); + la_movgr2fr_d(temp, rmask); + la_vext2xv_w_b(temp, temp); + la_xvbitsel_v(dest, src1, src2, temp); + } if (ir1_opnd_is_xmm(ir1_get_opnd(pir1, 0))) set_high128_xreg_to_zero(dest); diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index 938a04ff17..262ad39f00 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -83,6 +83,7 @@ if 'x86_64-linux-user' in target_dirs 'args': [ emulators['latx-x86_64'], files('../../avx-single-instruction-boundaries.asm'), + files('../../vblendpd-xmm-alias.asm'), files('../../avx128-shuffle.S'), files('../../avx-ymm-writeback.S'), files('../../vpshufd-distinct-regs.S'), diff --git a/tests/integration/vblendpd-xmm-alias.asm b/tests/integration/vblendpd-xmm-alias.asm new file mode 100644 index 0000000000..b747b297cb --- /dev/null +++ b/tests/integration/vblendpd-xmm-alias.asm @@ -0,0 +1,109 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro CHECK_RESULT 1 + vmovdqu [actual], ymm0 + lea rdx, [%1] + mov ecx, 4 +%%check: + mov rax, [actual + rcx * 8 - 8] + cmp rax, [rdx + rcx * 8 - 8] + jne fail + dec ecx + jnz %%check +%endmacro + +%macro PREPARE 1 + vmovdqu ymm1, [src1] + vmovdqu ymm2, [src2] +%if %1 == 0 + vmovdqu ymm0, [src2] +%endif +%endmacro + +_start: + PREPARE 0 + vblendpd xmm2, xmm1, xmm2, 0 + vmovdqu ymm0, ymm2 + CHECK_RESULT expected0 + + PREPARE 0 + vblendpd xmm2, xmm1, xmm2, 1 + vmovdqu ymm0, ymm2 + CHECK_RESULT expected1 + + PREPARE 0 + vblendpd xmm2, xmm1, xmm2, 2 + vmovdqu ymm0, ymm2 + CHECK_RESULT expected2 + + PREPARE 0 + vblendpd xmm2, xmm1, xmm2, 3 + vmovdqu ymm0, ymm2 + CHECK_RESULT expected3 + + PREPARE 0 + vblendpd xmm1, xmm1, xmm2, 0 + vmovdqu ymm0, ymm1 + CHECK_RESULT expected0 + + PREPARE 0 + vblendpd xmm1, xmm1, xmm2, 1 + vmovdqu ymm0, ymm1 + CHECK_RESULT expected1 + + PREPARE 0 + vblendpd xmm1, xmm1, xmm2, 2 + vmovdqu ymm0, ymm1 + CHECK_RESULT expected2 + + PREPARE 0 + vblendpd xmm1, xmm1, xmm2, 3 + vmovdqu ymm0, ymm1 + CHECK_RESULT expected3 + + PREPARE 0 + vblendpd xmm1, xmm1, xmm1, 1 + vmovdqu ymm0, ymm1 + CHECK_RESULT expected0 + + PREPARE 1 + vblendpd xmm0, xmm1, xmm2, 1 + CHECK_RESULT expected1 + + PREPARE 1 + vblendpd xmm0, xmm1, [src2], 2 + CHECK_RESULT expected2 + + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +src1: + dq 0x3ff0000000000000, 0x4000000000000000 + dq 0xaaaaaaaaaaaaaaaa, 0xbbbbbbbbbbbbbbbb +src2: + dq 0x4008000000000000, 0x4010000000000000 + dq 0xcccccccccccccccc, 0xdddddddddddddddd +expected0: + dq 0x3ff0000000000000, 0x4000000000000000, 0, 0 +expected1: + dq 0x4008000000000000, 0x4000000000000000, 0, 0 +expected2: + dq 0x3ff0000000000000, 0x4010000000000000, 0, 0 +expected3: + dq 0x4008000000000000, 0x4010000000000000, 0, 0 + +section .bss align=32 +actual: resb 32 + +section .note.GNU-stack noalloc noexec nowrite progbits From 56450899c2c96e14557bdf02e57125622436a8ce Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 31 Aug 2026 17:58:05 +0800 Subject: [PATCH 04/33] latx: reduce temporaries in AVX integer translation Remove intermediate copies from vector logic, shifts, packing, horizontal arithmetic, multiply-accumulate, carry-less multiply and SAD translation. Keep temporaries where overlapping sources or shift semantics require them. (cherry picked from commit b61cb010f5dc08332355692e8ac223b3edb4f629) Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx-shift.c | 33 +--- target/i386/latx/translator/tr-avx.c | 206 ++++++--------------- 2 files changed, 60 insertions(+), 179 deletions(-) diff --git a/target/i386/latx/translator/tr-avx-shift.c b/target/i386/latx/translator/tr-avx-shift.c index fc666fac30..b185cc6d37 100644 --- a/target/i386/latx/translator/tr-avx-shift.c +++ b/target/i386/latx/translator/tr-avx-shift.c @@ -150,8 +150,6 @@ bool translate_vpsrlx(IR1_INST * pir1) { IR2_INST * ( * rep_inst)(IR2_OPND, IR2_OPND); IR2_INST * ( * tr_inst_i)(IR2_OPND, IR2_OPND, int); IR2_INST * ( * tr_inst_r)(IR2_OPND, IR2_OPND, IR2_OPND); - IR2_OPND label_exit = ra_alloc_label(); - IR2_OPND label_shift = ra_alloc_label(); int max_count; switch (ir1_opcode(pir1)) { case dt_X86_INS_VPSRLW: @@ -192,18 +190,6 @@ bool translate_vpsrlx(IR1_INST * pir1) { IR2_OPND mask = ra_alloc_ftemp(); IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND count = ra_alloc_itemp(); - IR2_OPND max = ra_alloc_itemp(); - /* The low 64 bits of the count operand are one scalar shift count; - * a count equal to max_count still shifts, only a larger count - * yields zero. Compare against max_count + 1 to keep "==" valid. */ - la_addi_d(max, zero_ir2_opnd, max_count + 1); - la_vpickve2gr_d(count, src2, 0); - la_blt(count, max, label_shift); - la_xvxor_v(dest, dest, dest); - la_b(label_exit); - - la_label(label_shift); if (max_count == 63) { la_xvreplve0_d(mask, src2); la_xvldi(temp, VLDI_IMM_TYPE0(3, 63)); @@ -219,7 +205,6 @@ bool translate_vpsrlx(IR1_INST * pir1) { if (ir1_opnd_is_xmm(opnd0)) { set_high128_xreg_to_zero(dest); } - la_label(label_exit); return true; } @@ -267,21 +252,11 @@ bool translate_vpsrax(IR1_INST * pir1) { tr_inst_i(dest, src, imm); } else { IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND mask = ra_alloc_ftemp(); IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND temp_sign = ra_alloc_ftemp(); - if (max_count == 63) { - la_xvreplve0_d(mask, src2); - la_vldi(temp, VLDI_IMM_TYPE0(3, 63)); - la_xvsle_du(mask, mask, temp); - } else { - la_xvreplve0_d(mask, src2); - la_xvslei_du(mask, mask, max_count); - } - tr_inst_i(temp_sign, src, max_count); - rep_inst(temp, src2); - tr_inst_r(temp, src, temp); - la_xvbitsel_v(dest, temp_sign, temp, mask); + + la_xvmini_du(temp, src2, max_count); + rep_inst(temp, temp); + tr_inst_r(dest, src, temp); } if (ir1_opnd_is_xmm(opnd0)) { set_high128_xreg_to_zero(dest); diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 4eb5778836..3f033b2c05 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -762,10 +762,8 @@ bool translate_vandnpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vandn_v(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vandn_v(dest, src1, src2); set_high128_xreg_to_zero(dest); } return true; @@ -800,6 +798,7 @@ bool translate_vandpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); + la_vand_v(dest, src1, src2); set_high128_xreg_to_zero(dest); } @@ -835,10 +834,8 @@ bool translate_vorps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vor_v(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vor_v(dest, src1, src2); set_high128_xreg_to_zero(dest); } return true; @@ -873,10 +870,8 @@ bool translate_vxorps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vxor_v(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vxor_v(dest, src1, src2); set_high128_xreg_to_zero(dest); } return true; @@ -4037,32 +4032,17 @@ bool translate_vphsubw(IR1_INST * pir1) { IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); IR1_OPND * opnd2 = ir1_get_opnd(pir1, 2); - if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND dest = load_freg128_from_ir1(opnd0); - IR2_OPND src1 = load_freg128_from_ir1(opnd1); - IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - la_vhsubw_w_h(temp1, src1, src1); - la_vneg_h(temp1, temp1); - la_vhsubw_w_h(temp2, src2, src2); - la_vneg_h(temp2, temp2); + IR2_OPND dest = load_freg256_from_ir1(opnd0); + IR2_OPND src1 = load_freg256_from_ir1(opnd1); + IR2_OPND src2 = load_freg256_from_ir1(opnd2); + IR2_OPND even = ra_alloc_ftemp(); + IR2_OPND odd = ra_alloc_ftemp(); - la_vpickev_h(dest, temp2, temp1); + la_xvpickev_h(even, src2, src1); + la_xvpickod_h(odd, src2, src1); + la_xvsub_h(dest, even, odd); + if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); - } else { - IR2_OPND dest = load_freg256_from_ir1(opnd0); - IR2_OPND src1 = load_freg256_from_ir1(opnd1); - IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - la_xvhsubw_w_h(temp1, src1, src1); - la_xvneg_h(temp1, temp1); - la_xvhsubw_w_h(temp2, src2, src2); - la_xvneg_h(temp2, temp2); - - la_xvpickev_h(dest, temp2, temp1); - } return true; } @@ -4074,32 +4054,17 @@ bool translate_vphsubd(IR1_INST * pir1) { IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); IR1_OPND * opnd2 = ir1_get_opnd(pir1, 2); - if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND dest = load_freg128_from_ir1(opnd0); - IR2_OPND src1 = load_freg128_from_ir1(opnd1); - IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - la_vhsubw_d_w(temp1, src1, src1); - la_vneg_w(temp1, temp1); - la_vhsubw_d_w(temp2, src2, src2); - la_vneg_w(temp2, temp2); + IR2_OPND dest = load_freg256_from_ir1(opnd0); + IR2_OPND src1 = load_freg256_from_ir1(opnd1); + IR2_OPND src2 = load_freg256_from_ir1(opnd2); + IR2_OPND even = ra_alloc_ftemp(); + IR2_OPND odd = ra_alloc_ftemp(); - la_vpickev_w(dest, temp2, temp1); + la_xvpickev_w(even, src2, src1); + la_xvpickod_w(odd, src2, src1); + la_xvsub_w(dest, even, odd); + if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); - } else { - IR2_OPND dest = load_freg256_from_ir1(opnd0); - IR2_OPND src1 = load_freg256_from_ir1(opnd1); - IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - la_xvhsubw_d_w(temp1, src1, src1); - la_xvneg_w(temp1, temp1); - la_xvhsubw_d_w(temp2, src2, src2); - la_xvneg_w(temp2, temp2); - - la_xvpickev_w(dest, temp2, temp1); - } return true; } @@ -4111,45 +4076,17 @@ bool translate_vphsubsw(IR1_INST * pir1) { IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); IR1_OPND * opnd2 = ir1_get_opnd(pir1, 2); - if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND dest = load_freg128_from_ir1(opnd0); - IR2_OPND src1 = load_freg128_from_ir1(opnd1); - IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND temp3 = ra_alloc_ftemp(); - IR2_OPND temp4 = ra_alloc_ftemp(); - - la_vpickev_h(temp1, src1, src1); - la_vpickod_h(temp2, src1, src1); - la_vssub_h(temp3, temp1, temp2); //temp1-temp2 - - la_vpickev_h(temp1, src2, src2); - la_vpickod_h(temp2, src2, src2); - la_vssub_h(temp4, temp1, temp2); //temp1-temp2 + IR2_OPND dest = load_freg256_from_ir1(opnd0); + IR2_OPND src1 = load_freg256_from_ir1(opnd1); + IR2_OPND src2 = load_freg256_from_ir1(opnd2); + IR2_OPND even = ra_alloc_ftemp(); + IR2_OPND odd = ra_alloc_ftemp(); - la_vshuf4i_d(temp4, temp3, 0b00000110); - la_vori_b(dest, temp4, 0x0); + la_xvpickev_h(even, src2, src1); + la_xvpickod_h(odd, src2, src1); + la_xvssub_h(dest, even, odd); + if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); - } else { - IR2_OPND dest = load_freg256_from_ir1(opnd0); - IR2_OPND src1 = load_freg256_from_ir1(opnd1); - IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND temp3 = ra_alloc_ftemp(); - IR2_OPND temp4 = ra_alloc_ftemp(); - la_xvpickev_h(temp1, src1, src1); - la_xvpickod_h(temp2, src1, src1); - la_xvssub_h(temp3, temp1, temp2); //temp1-temp2 - - la_xvpickev_h(temp1, src2, src2); - la_xvpickod_h(temp2, src2, src2); - la_xvssub_h(temp4, temp1, temp2); //temp1-temp2 - - la_xvshuf4i_d(temp4, temp3, 0b00000110); - la_xvori_b(dest, temp4, 0x0); - } return true; } @@ -4446,10 +4383,8 @@ bool translate_vpsllvd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(opnd2); IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); la_vslei_wu(temp, src2, 31); - la_vand_v(temp2, temp, src2); - la_vsll_w(dest, src1, temp2); + la_vsll_w(dest, src1, src2); la_vand_v(dest, dest, temp); set_high128_xreg_to_zero(dest); } else if (ir1_opnd_is_ymm(opnd0)) { @@ -4458,10 +4393,8 @@ bool translate_vpsllvd(IR1_INST * pir1) { IR2_OPND src2 = load_freg256_from_ir1(opnd2); IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); la_xvslei_wu(temp, src2, 31); - la_xvand_v(temp2, temp, src2); - la_xvsll_w(dest, src1, temp2); + la_xvsll_w(dest, src1, src2); la_xvand_v(dest, dest, temp); } @@ -4482,13 +4415,10 @@ bool translate_vpsllvq(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(opnd2); IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - la_vandi_b(temp,temp,0); la_vldi(temp,0b0110000111111); la_vsle_du(temp, src2, temp); - la_vand_v(temp2, temp, src2); - la_vsll_d(dest, src1, temp2); + la_vsll_d(dest, src1, src2); la_vand_v(dest, dest, temp); set_high128_xreg_to_zero(dest); } else if (ir1_opnd_is_ymm(opnd0)) { @@ -4497,13 +4427,10 @@ bool translate_vpsllvq(IR1_INST * pir1) { IR2_OPND src2 = load_freg256_from_ir1(opnd2); IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - la_xvandi_b(temp,temp,0); la_xvldi(temp,0b0110000111111); la_xvsle_du(temp, src2, temp); - la_xvand_v(temp2, temp, src2); - la_xvsll_d(dest, src1, temp2); + la_xvsll_d(dest, src1, src2); la_xvand_v(dest, dest, temp); } @@ -4523,34 +4450,20 @@ bool translate_vpsravd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp01 = ra_alloc_ftemp(); - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND temp1 = ra_alloc_ftemp(); + IR2_OPND count = ra_alloc_ftemp(); - la_vclz_w(temp01, src1); - la_vseqi_w(temp01, temp01, 0x0); - - la_vslei_wu(temp, src2, 31); - la_vand_v(temp1, temp, src2); - la_vsra_w(temp1, src1, temp1); - la_vbitsel_v(dest, temp01, temp1, temp); + la_vmini_wu(count, src2, 31); + la_vsra_w(dest, src1, count); set_high128_xreg_to_zero(dest); } else if (ir1_opnd_is_ymm(opnd0)) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND temp01 = ra_alloc_ftemp(); - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND temp1 = ra_alloc_ftemp(); + IR2_OPND count = ra_alloc_ftemp(); - la_xvclz_w(temp01, src1); - la_xvseqi_w(temp01, temp01, 0x0); - - la_xvslei_wu(temp, src2, 31); - la_xvand_v(temp1, temp, src2); - la_xvsra_w(temp1, src1, temp1); - la_xvbitsel_v(dest, temp01, temp1, temp); + la_xvmini_wu(count, src2, 31); + la_xvsra_w(dest, src1, count); } return true; @@ -4773,20 +4686,15 @@ bool translate_vphminposuw(IR1_INST * pir1) { IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - la_xvori_b(temp, src, 0x0); - la_vsrli_d(temp, temp, 0x10); - la_vmin_hu(temp, temp, src); - la_vsrli_d(temp, temp, 0x10); - la_vmin_hu(temp, temp, src); - la_vsrli_d(temp, temp, 0x10); - la_vmin_hu(temp, temp, src); - - la_xvpickve_d(temp2, temp, 1); - la_vmin_hu(temp, temp, temp2); + la_vshuf4i_h(temp1, src, 0xb1); + la_vmin_hu(temp, temp1, src); + la_vshuf4i_h(temp1, temp, 0x4e); + la_vmin_hu(temp, temp, temp1); + la_vreplvei_d(temp1, temp, 1); + la_vmin_hu(temp, temp, temp1); la_xvreplve0_h(temp, temp); la_vseq_h(temp1, src, temp); - la_vandi_b(temp2, temp2, 0x0); - la_vfrstp_h(temp2, temp1, temp2); + la_vfrstpi_h(temp2, temp1, 0); la_vpackev_h(dest, temp2, temp); la_xvpickve_w(dest, dest, 0); return true; @@ -5264,7 +5172,6 @@ bool translate_vmpsadbw(IR1_INST *pir1) IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); IR2_OPND temp3 = ra_alloc_ftemp(); - IR2_OPND temp_dest = ra_alloc_ftemp(); la_vreplvei_w(temp1, src2, imm & 3); la_vmepatmsk_v(temp2, 1, imm & 0x4); @@ -5276,13 +5183,15 @@ bool translate_vmpsadbw(IR1_INST *pir1) la_vhaddw_hu_bu(temp2, temp2, temp2); la_vhaddw_wu_hu(temp2, temp2, temp2); la_vhaddw_hu_bu(temp3, temp3, temp3); - la_vhaddw_wu_hu(temp_dest, temp3, temp3); - la_vsrlni_h_w(temp_dest, temp2, 0); if(ir1_opnd_is_xmm(opnd0)){ - la_xvori_b(dest, temp_dest, 0); + la_vhaddw_wu_hu(dest, temp3, temp3); + la_vsrlni_h_w(dest, temp2, 0); set_high128_xreg_to_zero(dest); return true; } + IR2_OPND temp_dest = ra_alloc_ftemp(); + la_vhaddw_wu_hu(temp_dest, temp3, temp3); + la_vsrlni_h_w(temp_dest, temp2, 0); imm = imm>>3; la_vmepatmsk_v(temp2, 1, imm & 0x4); la_vmepatmsk_v(temp3, 2, imm & 0x4); @@ -6002,7 +5911,6 @@ bool translate_vpclmulqdq(IR1_INST * pir1) { } else { IR2_OPND src1 = ra_alloc_xmm(s1); IR2_OPND src2; - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND ftemp = ra_alloc_ftemp(); if (!ir1_opnd_is_mem(opnd2)) { @@ -6028,13 +5936,11 @@ bool translate_vpclmulqdq(IR1_INST * pir1) { ra_free_temp(ctrlp); emit_pclmul_ctz_loop(lhs, rhs, res_lo, res_hi); - la_vxor_v(temp, temp, temp); - la_vinsgr2vr_d(temp, res_lo, 0); - la_vinsgr2vr_d(temp, res_hi, 1); - la_xvori_b(dest, temp, 0); + la_vxor_v(dest, dest, dest); + la_vinsgr2vr_d(dest, res_lo, 0); + la_vinsgr2vr_d(dest, res_hi, 1); set_high128_xreg_to_zero(dest); ra_free_temp_auto(src2); - ra_free_temp(temp); ra_free_temp(ftemp); ra_free_temp(lhs); ra_free_temp(rhs); From 3ba8e5ed56f7f914374ee636a8d5480213617c2e Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Thu, 29 Oct 2026 17:27:43 +0800 Subject: [PATCH 05/33] LATX, opt: Remove redundant copies and inserts in AVX translation Write XMM VPANDN results directly and remove duplicate YMM VPBROADCASTQ inserts. Skip the src1 copy in VRCPSS and VRSQRTSS when dest is src1. For XMM VPBLENDD, select a base source and insert only the required words instead of building a vector mask. Reuse dest when it already holds a source, and load memory operands even when the mask selects no memory lanes. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx.c | 43 ++++----- .../avx-single-instruction-boundaries.asm | 87 +++++++++++++++++++ 2 files changed, 110 insertions(+), 20 deletions(-) diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 3f033b2c05..e964b29f9b 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -1987,9 +1987,7 @@ bool translate_vpandn(IR1_INST * pir1) { la_xvandn_v(dest, src1, src2); return true; } - IR2_OPND temp = ra_alloc_ftemp(); - la_xvandn_v(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vandn_v(dest, src1, src2); set_high128_xreg_to_zero(dest); return true; } @@ -2010,30 +2008,33 @@ bool translate_vpblendd(IR1_INST * pir1) { IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(opnd2); uint8_t imm = ir1_opnd_uimm(opnd3); - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND rmask = ra_alloc_itemp(); if (ir1_opnd_is_xmm(opnd0)) { + bool dest_is_src1 = ir1_opnd_is_same_reg(opnd0, opnd1); + bool dest_is_src2 = ir1_opnd_is_same_reg(opnd0, opnd2); if((imm & 0xf) == 0xf) { - la_xvori_b(dest, src2, 0); + if (!dest_is_src2) + la_xvori_b(dest, src2, 0); set_high128_xreg_to_zero(dest); return true; } else if((imm & 0xf) == 0x0){ - la_xvori_b(dest, src1, 0); + if (!dest_is_src1) + la_xvori_b(dest, src1, 0); set_high128_xreg_to_zero(dest); return true; } - uint64_t mask = 0; + bool base_is_src2 = dest_is_src2 || + (!dest_is_src1 && __builtin_popcount(imm & 0xf) > 2); + uint8_t insert_mask = (base_is_src2 ? ~imm : imm) & 0xf; + IR2_OPND base = base_is_src2 ? src2 : src1; + IR2_OPND insert_src = base_is_src2 ? src1 : src2; + + if (!dest_is_src1 && !dest_is_src2) + la_xvori_b(dest, base, 0); for (int i = 0; i < 4; i++) { - if (imm & (1 << i)) { - mask |= (0xFFULL) << (i * 8); - } + if (insert_mask & (1u << i)) + la_vextrins_w(dest, insert_src, VEXTRINS_IMM_4_0(i, i)); } - li_d(rmask, mask); - la_movgr2fr_d(temp, rmask); - la_vext2xv_w_b(temp, temp); - la_xvbitsel_v(dest, src1, src2, temp); - set_high128_xreg_to_zero(dest); return true; } @@ -2044,6 +2045,8 @@ bool translate_vpblendd(IR1_INST * pir1) { la_xvori_b(dest, src1, 0); return true; } + IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND rmask = ra_alloc_itemp(); uint64_t mask = 0; for (int i = 0; i < 8; i++) { if (imm & (1 << i)) { @@ -3523,8 +3526,6 @@ bool translate_vpbroadcastq(IR1_INST *pir1) set_high128_xreg_to_zero(dest); } else if (ir1_opnd_is_ymm(opnd0)) { la_xvreplve0_d(dest, src); - la_xvinsve0_d(dest, dest, 2); - la_xvinsve0_d(dest, dest, 3); } else { lsassert(0); } @@ -3759,7 +3760,8 @@ bool translate_vrsqrtss(IR1_INST * pir1) { /* this x86 instruction has no exception ,so we need mask all exception */ la_frsqrt_s(temp, src2); - la_xvori_b(dest, src1, 0x0); + if (!ir2_opnd_cmp(&dest, &src1)) + la_xvori_b(dest, src1, 0x0); la_xvinsve0_w(dest, temp, 0); set_high128_xreg_to_zero(dest); @@ -3820,7 +3822,8 @@ bool translate_vrcpss(IR1_INST * pir1) { /* this x86 instruction has no exception ,so we need mask all exception */ la_frecip_s(temp, src2); - la_xvori_b(dest, src1, 0x0); + if (!ir2_opnd_cmp(&dest, &src1)) + la_xvori_b(dest, src1, 0x0); la_xvinsve0_w(dest, temp, 0); set_high128_xreg_to_zero(dest); diff --git a/tests/integration/avx-single-instruction-boundaries.asm b/tests/integration/avx-single-instruction-boundaries.asm index 759440e374..b137469bae 100644 --- a/tests/integration/avx-single-instruction-boundaries.asm +++ b/tests/integration/avx-single-instruction-boundaries.asm @@ -45,6 +45,39 @@ section .text %endif %endmacro +%macro CHECK_SCALAR_COPY 1 + vmovdqu ymm0, [junk] + vmovdqu ymm1, [scalar_input] + %1 xmm0, xmm1, xmm1 + vmovdqu [scalar_reference], ymm0 + vmovdqu ymm1, [scalar_input] + %1 xmm1, xmm1, xmm1 + vmovdqu ymm0, ymm1 + CHECK scalar_reference, 14 + vmovdqu ymm1, [scalar_input] + %1 xmm1, xmm1, [scalar_input] + vmovdqu ymm0, ymm1 + CHECK scalar_reference, 15 +%endmacro + +%macro BLEND 2 + vmovdqu ymm0, [junk] + vmovdqu ymm1, [words2] + vmovdqu ymm2, [words3] + %if %2 = 1 + vmovdqu ymm0, ymm1 + vpblendd xmm0, xmm0, xmm2, %1 + %elif %2 = 2 + vmovdqu ymm0, ymm2 + vpblendd xmm0, xmm1, xmm0, %1 + %elif %2 = 3 + vpblendd xmm0, xmm1, [words3], %1 + %else + vpblendd xmm0, xmm1, xmm2, %1 + %endif + CHECK blend_ %+ %1, 16 +%endmacro + _start: %assign alias 0 %rep 3 @@ -52,7 +85,38 @@ _start: MADD vpmaddwd, alias, words2, words3, maddwd, ymm MADD vpmaddubsw, alias, bytes255, bytes128, maddubsw, xmm MADD vpmaddubsw, alias, bytes255, bytes128, maddubsw, ymm + MADD vpandn, alias, words2, words3, andnot, xmm + MADD vpandn, alias, words2, words3, andnot, ymm %assign alias alias + 1 +%endrep + + vmovdqu ymm0, [junk] + vpandn xmm0, xmm0, xmm0 + CHECK zeros, 9 + vmovdqu ymm1, [words2] + vpandn xmm0, xmm1, [words3] + CHECK andnot_xmm, 10 + + vmovdqu ymm0, [junk] + vpbroadcastq ymm0, xmm0 + CHECK junk, 11 + vmovdqu ymm1, [broadcast_input] + vpbroadcastq ymm0, xmm1 + CHECK broadcast_q, 12 + vpbroadcastq ymm0, [broadcast_input] + CHECK broadcast_q, 13 + + CHECK_SCALAR_COPY vrcpss + CHECK_SCALAR_COPY vrsqrtss + +%assign blend_mask 0 +%rep 16 +%assign blend_alias 0 +%rep 4 + BLEND blend_mask, blend_alias +%assign blend_alias blend_alias + 1 +%endrep +%assign blend_mask blend_mask + 1 %endrep vmovdqu ymm1, [words2] @@ -94,6 +158,9 @@ maddwd_xmm: times 4 dd 12 maddubsw: times 16 dw 0x8000 maddubsw_xmm: times 8 dw 0x8000 times 8 dw 0 +andnot: times 16 dw 1 +andnot_xmm: times 8 dw 1 + times 8 dw 0 zeros: times 4 dq 0 ones16: times 16 dw 1 ones32: times 8 dd 1 @@ -102,8 +169,28 @@ count15: dq 15, 0 count31: dq 31, 0 count63: dq 63, 0 count64: dq 64, 0 +broadcast_input: dq 0x0123456789abcdef, 0xfedcba9876543210, 0, -1 +broadcast_q: times 4 dq 0x0123456789abcdef +scalar_input: dd 0x40800000, 0x11223344, 0x55667788, 0x99aabbcc + times 4 dd 0xdeadbeef +%assign blend_mask 0 +%rep 16 +blend_ %+ blend_mask: +%assign lane 0 +%rep 4 + %if blend_mask & (1 << lane) + dd 0x00030003 + %else + dd 0x00020002 + %endif +%assign lane lane + 1 +%endrep + times 4 dd 0 +%assign blend_mask blend_mask + 1 +%endrep section .bss align=32 actual: resb 32 +scalar_reference: resb 32 section .note.GNU-stack noalloc noexec nowrite progbits From 40fff53c7b19da8e4548da2f39935355056598d5 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Thu, 29 Oct 2026 19:44:17 +0800 Subject: [PATCH 06/33] LATX, opt: Generate only live flags for AVX scalar comparisons Use reduced-flag generation for VCOMISS, VCOMISD, VUCOMISS and VUCOMISD. Load memory operands and perform the floating comparison even when its flags are unused, so flag elimination retains faults and invalid operations. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx-cmp.c | 19 +++- .../integration/registrations/avx/meson.build | 7 ++ tests/integration/vcomis-flag-reduction.asm | 93 +++++++++++++++++++ 3 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 tests/integration/vcomis-flag-reduction.asm diff --git a/target/i386/latx/translator/tr-avx-cmp.c b/target/i386/latx/translator/tr-avx-cmp.c index f77dea6f59..d1e1e0dca8 100644 --- a/target/i386/latx/translator/tr-avx-cmp.c +++ b/target/i386/latx/translator/tr-avx-cmp.c @@ -22,6 +22,24 @@ static inline void xcomisx(IR1_INST *pir1, bool is_double, bool qnan_exp) lsassert(ir1_opnd_num(pir1) == 2); IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); +#ifdef CONFIG_LATX_XCOMISX_OPT + uint8_t eflags = ir1_get_eflag_def(pir1) & + ~lsenv->tr_data->curr_ir1_skipped_eflags; + if (eflags != 0x3f) { + /* Keep the comparison even when its flags die: it can raise invalid. */ + if (!(eflags & (ZF_USEDEF_BIT | PF_USEDEF_BIT | CF_USEDEF_BIT))) { + if (is_double) { + la_fcmp_cond_d(fcc0_ir2_opnd, dest, src, + FCMP_COND_CUEQ + qnan_exp); + } else { + la_fcmp_cond_s(fcc0_ir2_opnd, dest, src, + FCMP_COND_CUEQ + qnan_exp); + } + } + generate_xcomisx(dest, src, is_double, qnan_exp, eflags); + return; + } +#endif /* 0. set flag = 0 */ IR2_OPND flag_zf = ra_alloc_itemp(); IR2_OPND flag_pf = ra_alloc_itemp(); @@ -56,7 +74,6 @@ static inline void xcomisx(IR1_INST *pir1, bool is_double, bool qnan_exp) ra_free_temp(flag_pf); ra_free_temp(flag_zf); ra_free_temp(flag); - } bool translate_vcomisd(IR1_INST * pir1) { diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index 262ad39f00..761916e19b 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -53,6 +53,13 @@ if 'x86_64-linux-user' in target_dirs 'args': [emulators['latx-x86_64'], files('../../vcomis-jcc.S')], 'timeout': 120, }] + latx_integration_tests += [{ + 'name': 'test-vcomis-flag-reduction', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], files('../../vcomis-flag-reduction.asm')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 180, + }] latx_integration_tests += [{ 'name': 'test-xcomis-patterns', 'runner': find_program('../../test-avx-regressions.py'), diff --git a/tests/integration/vcomis-flag-reduction.asm b/tests/integration/vcomis-flag-reduction.asm new file mode 100644 index 0000000000..0c241c0f2a --- /dev/null +++ b/tests/integration/vcomis-flag-reduction.asm @@ -0,0 +1,93 @@ +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +; Each subset is consumed before XOR makes all remaining flags dead. +%macro CHECK 5 + vmovdqu ymm0, [%2] + vmovdqu ymm1, [%3] + %1 xmm0, xmm1 +%if %5 & 1 + setc [result] +%endif +%if %5 & 2 + setp [result + 1] +%endif +%if %5 & 4 + lahf + mov [result + 2], ah +%endif +%if %5 & 8 + setz [result + 3] +%endif +%if %5 & 16 + sets [result + 4] +%endif +%if %5 & 32 + seto [result + 5] +%endif + xor eax, eax +%if %5 & 1 + cmp byte [result], (%4 & 1) != 0 + jne fail +%endif +%if %5 & 2 + cmp byte [result + 1], (%4 & 4) != 0 + jne fail +%endif +%if %5 & 4 + cmp byte [result + 2], %4 | 2 + jne fail +%endif +%if %5 & 8 + cmp byte [result + 3], (%4 & 0x40) != 0 + jne fail +%endif +%if %5 & 16 + cmp byte [result + 4], 0 + jne fail +%endif +%if %5 & 32 + cmp byte [result + 5], 0 + jne fail +%endif +%endmacro + +%macro SUITE 2 +%assign mask 0 +%rep 64 + CHECK %1, two_%2, one_%2, 0, mask + CHECK %1, one_%2, two_%2, 1, mask + CHECK %1, one_%2, one_%2, 0x40, mask + CHECK %1, nan_%2, one_%2, 0x45, mask +%assign mask mask + 1 +%endrep +%endmacro + +section .text +_start: + ldmxcsr [clean] + SUITE vcomiss, s + SUITE vucomiss, s + SUITE vcomisd, d + SUITE vucomisd, d + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +one_s: dd 0x3f800000, 1, 2, 3, 4, 5, 6, 7 +two_s: dd 0x40000000, 8, 9, 10, 11, 12, 13, 14 +nan_s: dd 0x7fc00001, 15, 16, 17, 18, 19, 20, 21 +one_d: dq 0x3ff0000000000000, 22, 23, 24 +two_d: dq 0x4000000000000000, 25, 26, 27 +nan_d: dq 0x7ff8000000000001, 28, 29, 30 +clean: dd 0x1f80 +section .bss +result: resb 6 +section .note.GNU-stack noalloc noexec nowrite progbits From d16833d3f97fb59e3d3548d40a3306d1ac920aa9 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Thu, 29 Oct 2026 19:44:55 +0800 Subject: [PATCH 07/33] LATX, opt: Generate only live flags for AVX vector tests Mark VPTEST, VTESTPS and VTESTPD as writing all six arithmetic flags. Generate only the needed ZF and CF results and required constant flags, using conditional moves instead of branches around flag writes. Load operands before eliminating unused flags so memory faults are retained. Signed-off-by: Zhaoyang Xie --- target/i386/latx/include/translate.h | 2 + .../i386/latx/optimization/flag-reduction.c | 3 + target/i386/latx/translator/tr-avx.c | 227 +++++++----------- tests/integration/avx-test-flags.asm | 89 +++++++ .../integration/registrations/avx/meson.build | 7 + 5 files changed, 182 insertions(+), 146 deletions(-) create mode 100644 tests/integration/avx-test-flags.asm diff --git a/target/i386/latx/include/translate.h b/target/i386/latx/include/translate.h index 098f05867c..a8859c02f1 100644 --- a/target/i386/latx/include/translate.h +++ b/target/i386/latx/include/translate.h @@ -1119,6 +1119,8 @@ TRANS_FUNC_DEF(vpshufb_lsx); TRANS_FUNC_DEF(vpsubx); TRANS_FUNC_DEF(vptest); TRANS_FUNC_DEF(vptest_lsx); +void generate_avx_test_flags(IR1_INST *, IR2_OPND, IR2_OPND, IR2_OPND, + uint8_t); TRANS_FUNC_DEF(vpunpckhxx); TRANS_FUNC_DEF(vpunpckhxx_lsx); TRANS_FUNC_DEF(vpunpcklxx); diff --git a/target/i386/latx/optimization/flag-reduction.c b/target/i386/latx/optimization/flag-reduction.c index c8ea963909..516004bc77 100644 --- a/target/i386/latx/optimization/flag-reduction.c +++ b/target/i386/latx/optimization/flag-reduction.c @@ -116,6 +116,9 @@ static const IR1_EFLAG_USEDEF ir1_opcode_eflag_usedef[] = { FLAG_DEFINE(VCOMISS, __NONE, __ALL_EFLAGS, __NONE), FLAG_DEFINE(VUCOMISD, __NONE, __ALL_EFLAGS, __NONE), FLAG_DEFINE(VUCOMISS, __NONE, __ALL_EFLAGS, __NONE), + FLAG_DEFINE(VPTEST, __NONE, __ALL_EFLAGS, __NONE), + FLAG_DEFINE(VTESTPS, __NONE, __ALL_EFLAGS, __NONE), + FLAG_DEFINE(VTESTPD, __NONE, __ALL_EFLAGS, __NONE), FLAG_DEFINE(IRET, __NONE, __INVALID, __NONE), /* Jcc */ diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index e964b29f9b..e7d548dc1a 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -2554,52 +2554,92 @@ bool translate_vpsubx(IR1_INST * pir1) { return true; } -bool translate_vptest(IR1_INST * pir1) { - if (!option_enable_lasx) { - return translate_vptest_lsx(pir1); - } +/* Requested ZF/CF values also remain in fcc0/fcc1 for a fused branch. */ +void generate_avx_test_flags(IR1_INST *pir1, IR2_OPND dest, IR2_OPND src, + IR2_OPND flags, uint8_t eflags) +{ + bool wide = ir1_opnd_is_ymm(ir1_get_opnd(pir1, 0)); + int shift = ir1_opcode(pir1) == dt_X86_INS_VTESTPS ? 31 : + ir1_opcode(pir1) == dt_X86_INS_VTESTPD ? 63 : 0; - IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); - IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); - lsassert(ir1_opnd_is_xmm(opnd0) || ir1_opnd_is_ymm(opnd0)); - IR2_OPND dest = load_freg256_from_ir1(opnd0); - IR2_OPND src = load_freg256_from_ir1(opnd1); + if (!(eflags & CF_USEDEF_BIT)) { + la_mov64(flags, zero_ir2_opnd); + } + if (!(eflags & (ZF_USEDEF_BIT | CF_USEDEF_BIT))) { + return; + } IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND label_1 = ra_alloc_label(); - IR2_OPND label_2 = ra_alloc_label(); - IR2_OPND n4095_opnd = ra_alloc_num_4095(); - - if (ir1_opnd_is_xmm(opnd0)) { - la_x86mtflag(zero_ir2_opnd, 0x3f); - la_vand_v(temp, src, dest); - la_vseteqz_v(fcc0_ir2_opnd, temp); - la_bceqz(fcc0_ir2_opnd, label_1); - la_x86mtflag(n4095_opnd, ZF_USEDEF_BIT); - - la_label(label_1); - la_vandn_v(temp, dest, src); - la_vseteqz_v(fcc0_ir2_opnd, temp); - la_bceqz(fcc0_ir2_opnd, label_2); - la_x86mtflag(n4095_opnd, CF_USEDEF_BIT); - la_label(label_2); - } else { - la_x86mtflag(zero_ir2_opnd, 0x3f); - la_xvand_v(temp, src, dest); - la_xvseteqz_v(fcc0_ir2_opnd, temp); - la_bceqz(fcc0_ir2_opnd, label_1); - la_x86mtflag(n4095_opnd, ZF_USEDEF_BIT); + IR2_OPND bit = ra_alloc_itemp(); + for (int cf = 1; cf >= 0; cf--) { + if (!(eflags & (cf ? CF_USEDEF_BIT : ZF_USEDEF_BIT))) { + continue; + } + IR2_OPND condition = cf ? fcc1_ir2_opnd : fcc0_ir2_opnd; + if (wide) { + if (cf) { + la_xvandn_v(temp, dest, src); + } else { + la_xvand_v(temp, dest, src); + } + if (shift == 31) { + la_xvsrli_w(temp, temp, shift); + } else if (shift == 63) { + la_xvsrli_d(temp, temp, shift); + } + la_xvseteqz_v(condition, temp); + } else { + if (cf) { + la_vandn_v(temp, dest, src); + } else { + la_vand_v(temp, dest, src); + } + if (shift == 31) { + la_vsrli_w(temp, temp, shift); + } else if (shift == 63) { + la_vsrli_d(temp, temp, shift); + } + la_vseteqz_v(condition, temp); + } + if (cf) { + la_movcf2gr(flags, condition); + } else { + la_movcf2gr(bit, condition); + la_bstrins_w(flags, bit, ZF_BIT_INDEX, ZF_BIT_INDEX); + } + } + ra_free_temp(bit); + ra_free_temp(temp); +} - la_label(label_1); - la_xvandn_v(temp, dest, src); - la_xvseteqz_v(fcc0_ir2_opnd, temp); - la_bceqz(fcc0_ir2_opnd, label_2); - la_x86mtflag(n4095_opnd, CF_USEDEF_BIT); - la_label(label_2); +static bool translate_avx_test(IR1_INST *pir1) +{ + IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0); + IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); + bool wide = ir1_opnd_is_ymm(opnd0); + IR2_OPND dest = wide ? load_freg256_from_ir1(opnd0) : + load_freg128_from_ir1(opnd0); + IR2_OPND src = wide ? load_freg256_from_ir1(opnd1) : + load_freg128_from_ir1(opnd1); + uint8_t eflags = ir1_get_eflag_def(pir1) & + ~lsenv->tr_data->curr_ir1_skipped_eflags; + /* Operand loads must still fault when all generated flags are dead. */ + if (eflags) { + IR2_OPND flags = ra_alloc_itemp(); + generate_avx_test_flags(pir1, dest, src, flags, eflags); + la_x86mtflag(flags, eflags); + ra_free_temp(flags); } - ra_free_num_4095(n4095_opnd); return true; } +bool translate_vptest(IR1_INST *pir1) +{ + if (!option_enable_lasx) { + return translate_vptest_lsx(pir1); + } + return translate_avx_test(pir1); +} + bool translate_vpunpckhxx(IR1_INST * pir1) { if (!option_enable_lasx) { return translate_vpunpckhxx_lsx(pir1); @@ -4765,119 +4805,14 @@ bool translate_vtestps(IR1_INST * pir1) { if (!option_enable_lasx) { return translate_vtestps_lsx(pir1); } - - IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); - IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); - if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND dest = load_freg128_from_ir1(opnd0); - IR2_OPND src = load_freg128_from_ir1(opnd1); - - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND n4095_opnd = ra_alloc_num_4095(); - IR2_OPND label_1 = ra_alloc_label(); - IR2_OPND label_2 = ra_alloc_label(); - - la_x86mtflag(zero_ir2_opnd, 0x3f); - la_vand_v(temp, dest, src); - la_vsrli_w(temp, temp, 0x1f); - la_vseteqz_v(fcc0_ir2_opnd, temp); //temp==0 fcc=1; - la_bceqz(fcc0_ir2_opnd, label_1); //fcc==1,not jump - la_x86mtflag(n4095_opnd, ZF_USEDEF_BIT); - - la_label(label_1); - la_vandn_v(temp, dest, src); - la_vsrli_w(temp, temp, 0x1f); - la_vseteqz_v(fcc0_ir2_opnd, temp); //temp==0 fcc=1; - la_bceqz(fcc0_ir2_opnd, label_2); //fcc==1,not jump - la_x86mtflag(n4095_opnd, CF_USEDEF_BIT); - la_label(label_2); - - } else { - IR2_OPND dest = load_freg256_from_ir1(opnd0); - IR2_OPND src = load_freg256_from_ir1(opnd1); - - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND n4095_opnd = ra_alloc_num_4095(); - IR2_OPND label_1 = ra_alloc_label(); - IR2_OPND label_2 = ra_alloc_label(); - - la_x86mtflag(zero_ir2_opnd, 0x3f); - la_xvand_v(temp, dest, src); - la_xvsrli_w(temp, temp, 0x1f); - la_xvseteqz_v(fcc0_ir2_opnd, temp); //temp==0 fcc=1; - la_bceqz(fcc0_ir2_opnd, label_1); //fcc==1,not jump - la_x86mtflag(n4095_opnd, ZF_USEDEF_BIT); - - la_label(label_1); - la_xvandn_v(temp, dest, src); - la_xvsrli_w(temp, temp, 0x1f); - la_xvseteqz_v(fcc0_ir2_opnd, temp); //temp==0 fcc=1; - la_bceqz(fcc0_ir2_opnd, label_2); //fcc==1,not jump - la_x86mtflag(n4095_opnd, CF_USEDEF_BIT); - la_label(label_2); - } - - return true; + return translate_avx_test(pir1); } bool translate_vtestpd(IR1_INST * pir1) { if (!option_enable_lasx) { return translate_vtestpd_lsx(pir1); } - - IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); - IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); - if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND dest = load_freg128_from_ir1(opnd0); - IR2_OPND src = load_freg128_from_ir1(opnd1); - - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND n4095_opnd = ra_alloc_num_4095(); - IR2_OPND label_1 = ra_alloc_label(); - IR2_OPND label_2 = ra_alloc_label(); - - la_x86mtflag(zero_ir2_opnd, 0x3f); - la_vand_v(temp, dest, src); - la_vsrli_d(temp, temp, 0x3f); - la_vseteqz_v(fcc0_ir2_opnd, temp); //temp==0 fcc=1; - la_bceqz(fcc0_ir2_opnd, label_1); //fcc==1,not jump - la_x86mtflag(n4095_opnd, ZF_USEDEF_BIT); - - la_label(label_1); - la_vandn_v(temp, dest, src); - la_vsrli_d(temp, temp, 0x3f); - la_vseteqz_v(fcc0_ir2_opnd, temp); //temp==0 fcc=1; - la_bceqz(fcc0_ir2_opnd, label_2); //fcc==1,not jump - la_x86mtflag(n4095_opnd, CF_USEDEF_BIT); - la_label(label_2); - - } else { - IR2_OPND dest = load_freg256_from_ir1(opnd0); - IR2_OPND src = load_freg256_from_ir1(opnd1); - - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND n4095_opnd = ra_alloc_num_4095(); - IR2_OPND label_1 = ra_alloc_label(); - IR2_OPND label_2 = ra_alloc_label(); - - la_x86mtflag(zero_ir2_opnd, 0x3f); - la_xvand_v(temp, dest, src); - la_xvsrli_d(temp, temp, 0x3f); - la_xvseteqz_v(fcc0_ir2_opnd, temp); //temp==0 fcc=1; - la_bceqz(fcc0_ir2_opnd, label_1); //fcc==1,not jump - la_x86mtflag(n4095_opnd, ZF_USEDEF_BIT); - - la_label(label_1); - la_xvandn_v(temp, dest, src); - la_xvsrli_d(temp, temp, 0x3f); - la_xvseteqz_v(fcc0_ir2_opnd, temp); //temp==0 fcc=1; - la_bceqz(fcc0_ir2_opnd, label_2); //fcc==1,not jump - la_x86mtflag(n4095_opnd, CF_USEDEF_BIT); - la_label(label_2); - - } - - return true; + return translate_avx_test(pir1); } bool translate_vpackssxx(IR1_INST * pir1) { diff --git a/tests/integration/avx-test-flags.asm b/tests/integration/avx-test-flags.asm new file mode 100644 index 0000000000..b13f747a50 --- /dev/null +++ b/tests/integration/avx-test-flags.asm @@ -0,0 +1,89 @@ +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +%macro CHECK 6 + vmovdqu ymm0, [%3] + vmovdqu ymm1, [%4] + %1 %2 %+ 0, %2 %+ 1 +%if %6 & 1 + setc [result] +%endif +%if %6 & 2 + setp [result + 1] +%endif +%if %6 & 4 + lahf + mov [result + 2], ah +%endif +%if %6 & 8 + setz [result + 3] +%endif +%if %6 & 16 + sets [result + 4] +%endif +%if %6 & 32 + seto [result + 5] +%endif + xor eax, eax +%if %6 & 1 + cmp byte [result], (%5 & 1) != 0 + jne fail +%endif +%if %6 & 2 + cmp byte [result + 1], 0 + jne fail +%endif +%if %6 & 4 + cmp byte [result + 2], %5 | 2 + jne fail +%endif +%if %6 & 8 + cmp byte [result + 3], (%5 & 0x40) != 0 + jne fail +%endif +%if %6 & 16 + cmp byte [result + 4], 0 + jne fail +%endif +%if %6 & 32 + cmp byte [result + 5], 0 + jne fail +%endif +%endmacro + +%macro SUITE 2 +%assign mask 0 +%rep 64 + CHECK %1, %2, zeros, zeros, 0x41, mask + CHECK %1, %2, zeros, ones, 0x40, mask + CHECK %1, %2, ones, ones, 1, mask + CHECK %1, %2, half, ones, 0, mask +%assign mask mask + 1 +%endrep +%endmacro + +section .text +_start: + SUITE vptest, xmm + SUITE vptest, ymm + SUITE vtestps, xmm + SUITE vtestps, ymm + SUITE vtestpd, xmm + SUITE vtestpd, ymm + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +zeros: times 4 dq 0 +ones: times 4 dq -1 +half: dq -1, 0, -1, 0 +section .bss +result: resb 6 +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index 761916e19b..49441901f6 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -60,6 +60,13 @@ if 'x86_64-linux-user' in target_dirs 'env': {'LATX_SOFTFPU': '0'}, 'timeout': 180, }] + latx_integration_tests += [{ + 'name': 'test-avx-test-flags', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], files('../../avx-test-flags.asm')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 180, + }] latx_integration_tests += [{ 'name': 'test-xcomis-patterns', 'runner': find_program('../../test-avx-regressions.py'), From e8c3a5856b81b1e1bc274580bfce30d2085113ed Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Thu, 29 Oct 2026 19:45:21 +0800 Subject: [PATCH 08/33] LATX, opt: Branch directly on AVX vector test results Fuse adjacent VPTEST, VTESTPS or VTESTPD with JE, JNE, JB, JAE, JBE or JA. Use the vector test conditions directly instead of reading EFLAGS back. Write any live flags before leaving the block. Use ordinary translation for two-successor TU blocks and when the pattern option or LASX is disabled. Signed-off-by: Zhaoyang Xie --- target/i386/latx/optimization/insts-pattern.c | 26 +++++ target/i386/latx/translator/tr-pattern.c | 95 +++++++++++++++++++ tests/integration/avx-test-jcc.asm | 66 +++++++++++++ .../integration/registrations/avx/meson.build | 7 ++ 4 files changed, 194 insertions(+) create mode 100644 tests/integration/avx-test-jcc.asm diff --git a/target/i386/latx/optimization/insts-pattern.c b/target/i386/latx/optimization/insts-pattern.c index 7ebeafd1df..6c1d5c915d 100644 --- a/target/i386/latx/optimization/insts-pattern.c +++ b/target/i386/latx/optimization/insts-pattern.c @@ -443,6 +443,32 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i IR1_OPND *opnd0 = NULL; IR1_OPND *opnd1 = NULL; switch (ir1_opcode(pir1)) { +#ifdef CONFIG_LATX_AVX_OPT + case WRAP(VPTEST): + case WRAP(VTESTPS): + case WRAP(VTESTPD): + SCAN_CHECK(scan, 0); + ir1_jcc = SCAN_IR1(tb, scan, 0); + if (!option_enable_lasx || pir1_index + 1 != SCAN_IDX(scan, 0)) { + return false; + } + switch (ir1_opcode(ir1_jcc)) { + case WRAP(JE): + case WRAP(JNE): + case WRAP(JB): + case WRAP(JAE): + case WRAP(JBE): + case WRAP(JA): + instptn_check_test_jcc_0(); + pir1->instptn.opc = INSTPTN_OPC_TEST_JCC; + pir1->instptn.next = ir1_jcc; + ir1_jcc->instptn.opc = INSTPTN_OPC_NOP; + break; + default: + break; + } + return false; +#endif case WRAP(CMP): SCAN_CHECK(scan, 0); ir1_jcc = SCAN_IR1(tb, scan, 0); diff --git a/target/i386/latx/translator/tr-pattern.c b/target/i386/latx/translator/tr-pattern.c index ee15720703..70d85d9bf1 100644 --- a/target/i386/latx/translator/tr-pattern.c +++ b/target/i386/latx/translator/tr-pattern.c @@ -2379,6 +2379,94 @@ static bool translate_ucomiss_xx_jcc(IR1_INST *pir1) #endif +#ifdef CONFIG_LATX_AVX_OPT +static bool translate_avx_test_jcc(IR1_INST *pir1) +{ + IR1_INST *next = pir1->instptn.next; +#ifdef CONFIG_LATX_TU + TranslationBlock *tb = lsenv->tr_data->curr_tb; + if (tb->s_data->next_tb[TU_TB_INDEX_NEXT] && + tb->s_data->next_tb[TU_TB_INDEX_TARGET]) { + switch (ir1_opcode(pir1)) { + case WRAP(VPTEST): + translate_vptest(pir1); + break; + case WRAP(VTESTPS): + translate_vtestps(pir1); + break; + case WRAP(VTESTPD): + translate_vtestpd(pir1); + break; + default: + lsassert(0); + } + return translate_jcc(next); + } +#endif + IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0); + IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); + bool wide = ir1_opnd_is_ymm(opnd0); + IR2_OPND dest = wide ? load_freg256_from_ir1(opnd0) : + load_freg128_from_ir1(opnd0); + IR2_OPND src = wide ? load_freg256_from_ir1(opnd1) : + load_freg128_from_ir1(opnd1); + IR2_OPND flags = ra_alloc_itemp(); + IR2_OPND cond = ra_alloc_itemp(); + IR2_OPND target = ra_alloc_label(); + uint8_t eflags = ir1_get_eflag_def(pir1) & + ~lsenv->tr_data->curr_ir1_skipped_eflags; + int mask = ZF_BIT; + bool nonzero = true; + switch (ir1_opcode(next)) { + case WRAP(JNE): + nonzero = false; + break; + case WRAP(JAE): + nonzero = false; + /* fall through */ + case WRAP(JB): + mask = CF_BIT; + break; + case WRAP(JA): + nonzero = false; + /* fall through */ + case WRAP(JBE): + mask = ZF_BIT | CF_BIT; + break; + case WRAP(JE): + break; + default: + lsassert(0); + } + generate_avx_test_flags(pir1, dest, src, flags, + eflags | ir1_get_eflag_use(next)); + if (eflags) { + la_x86mtflag(flags, eflags); + } + if (mask == (ZF_BIT | CF_BIT)) { + la_andi(cond, flags, mask); + if (nonzero) { + la_bne(cond, zero_ir2_opnd, target); + } else { + la_beq(cond, zero_ir2_opnd, target); + } + } else { + IR2_OPND condition = mask == ZF_BIT ? fcc0_ir2_opnd : fcc1_ir2_opnd; + if (nonzero) { + la_bcnez(condition, target); + } else { + la_bceqz(condition, target); + } + } + ra_free_temp(cond); + ra_free_temp(flags); + tr_generate_exit_tb(next, 0); + la_label(target); + tr_generate_exit_tb(next, 1); + return true; +} +#endif + bool try_translate_instptn(IR1_INST *pir1) { instptn_check_false(); @@ -2392,6 +2480,13 @@ bool try_translate_instptn(IR1_INST *pir1) case INSTPTN_OPC_CMP_JCC: return translate_cmp_jcc(pir1); case INSTPTN_OPC_TEST_JCC: +#ifdef CONFIG_LATX_AVX_OPT + if (ir1_opcode(pir1) == WRAP(VPTEST) || + ir1_opcode(pir1) == WRAP(VTESTPS) || + ir1_opcode(pir1) == WRAP(VTESTPD)) { + return translate_avx_test_jcc(pir1); + } +#endif return translate_test_jcc(pir1); case INSTPTN_OPC_CQO_IDIV: return translate_cqo_idiv(pir1); diff --git a/tests/integration/avx-test-jcc.asm b/tests/integration/avx-test-jcc.asm new file mode 100644 index 0000000000..f3cb7a25d7 --- /dev/null +++ b/tests/integration/avx-test-jcc.asm @@ -0,0 +1,66 @@ +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +; Check both the branch outcome and flags visible in its successor. +%macro CHECK 7 + vmovdqu ymm0, [%3] + vmovdqu ymm1, [%4] + %1 %2 %+ 0, %2 %+ 1 + %5 %%taken + mov r9d, 0 + jmp %%flags +%%taken: + mov r9d, 1 +%%flags: + pushfq + pop rax + and eax, 0x8d5 + cmp eax, %6 + jne fail + cmp r9d, (%7) & 1 + jne fail +%endmacro + +%macro RELATION 5 + CHECK %1, %2, %3, %4, je, %5, (%5 & 0x40) != 0 + CHECK %1, %2, %3, %4, jne, %5, (%5 & 0x40) = 0 + CHECK %1, %2, %3, %4, jb, %5, (%5 & 1) != 0 + CHECK %1, %2, %3, %4, jae, %5, (%5 & 1) = 0 + CHECK %1, %2, %3, %4, jbe, %5, (%5 & 0x41) != 0 + CHECK %1, %2, %3, %4, ja, %5, (%5 & 0x41) = 0 +%endmacro + +%macro SUITE 2 + RELATION %1, %2, zeros, zeros, 0x41 + RELATION %1, %2, zeros, ones, 0x40 + RELATION %1, %2, ones, ones, 1 + RELATION %1, %2, half, ones, 0 +%endmacro + +section .text +_start: + mov ecx, 128 +again: + SUITE vptest, xmm + SUITE vptest, ymm + SUITE vtestps, xmm + SUITE vtestps, ymm + SUITE vtestpd, xmm + SUITE vtestpd, ymm + dec ecx + jnz again + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +zeros: times 4 dq 0 +ones: times 4 dq -1 +half: dq -1, 0, -1, 0 +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index 49441901f6..f92ec87d04 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -67,6 +67,13 @@ if 'x86_64-linux-user' in target_dirs 'env': {'LATX_SOFTFPU': '0'}, 'timeout': 180, }] + latx_integration_tests += [{ + 'name': 'test-avx-test-jcc', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], files('../../avx-test-jcc.asm')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 180, + }] latx_integration_tests += [{ 'name': 'test-xcomis-patterns', 'runner': find_program('../../test-avx-regressions.py'), From 233501c701b056660a1db05c618cc2f20ca089f0 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Fri, 30 Oct 2026 10:28:10 +0800 Subject: [PATCH 09/33] LATX, opt: Skip FCSR writes for masked fixed VROUNDSD When imm[3:2] is clear and all native traps are masked, use one fixed-direction round for both the result and exception status. Remove the preliminary round and the FCSR save/mask/restore sequence from this path. Use the existing sequence when traps are enabled, precision is suppressed or MXCSR selects the rounding direction. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx.c | 28 +++- .../integration/registrations/avx/meson.build | 1 + tests/integration/vroundsd-fixed.asm | 128 ++++++++++++++++++ 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 tests/integration/vroundsd-fixed.asm diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index e7d548dc1a..78b6a5d4ce 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -5644,6 +5644,31 @@ bool translate_vroundsd(IR1_INST *pir1) IR2_OPND mxcsr = ra_alloc_itemp(); bool is_xmm = ir1_opnd_is_xmm(opnd0); la_xvreplve0_d(src, src2); + IR2_OPND merge = ra_alloc_label(); + if (!(imm & 0xc)) { + IR2_OPND slow = ra_alloc_label(); + + /* With traps masked, the fixed round also supplies exception flags. */ + la_movfcsr2gr(fcsr, fcsr_ir2_opnd); + la_andi(fcsr, fcsr, FCSR_ENABLE_SET); + la_bnez(fcsr, slow); + switch (imm & 0x3) { + case 0: + la_vfrintrne_d(temp_dest, src); + break; + case 1: + la_vfrintrm_d(temp_dest, src); + break; + case 2: + la_vfrintrp_d(temp_dest, src); + break; + default: + la_vfrintrz_d(temp_dest, src); + break; + } + la_b(merge); + la_label(slow); + } if(imm & 0x8){ IR2_INST * ( * tr_inst)(IR2_OPND, IR2_OPND, IR2_OPND, int); tr_inst = is_xmm ? la_vfcmp_cond_d : la_xvfcmp_cond_d; @@ -5706,13 +5731,14 @@ bool translate_vroundsd(IR1_INST *pir1) tr_inst = is_xmm ? la_vfrintrz_d : la_xvfrintrz_d; tr_inst(temp_dest, src); } + la_movgr2fcsr(fcsr_ir2_opnd, fcsr_save); + la_label(merge); if (ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) { la_xvori_b(dest, src1, 0); } la_xvinsve0_d(dest, temp_dest, 0); set_high128_xreg_to_zero(dest); - la_movgr2fcsr(fcsr_ir2_opnd, fcsr_save); ra_free_temp(temp); ra_free_temp(fcsr); diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index f92ec87d04..cdd5aa0407 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -109,6 +109,7 @@ if 'x86_64-linux-user' in target_dirs files('../../avx-ymm-writeback.S'), files('../../vpshufd-distinct-regs.S'), files('../../vroundps-imm0b.S'), + files('../../vroundsd-fixed.asm'), files('../../vcvtdq2pd-ymm-alias.asm'), files('../../avx-regression-extract-alias.S'), files('../../avx-regression-narrow-state.S'), diff --git a/tests/integration/vroundsd-fixed.asm b/tests/integration/vroundsd-fixed.asm new file mode 100644 index 0000000000..5552e8ff7f --- /dev/null +++ b/tests/integration/vroundsd-fixed.asm @@ -0,0 +1,128 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro VERIFY 2 + vmovdqu [output], %1 + mov rax, [output] + cmp rax, [rdi + %2 * 8] + jne fail + mov rax, [output + 8] + cmp rax, [rsi + 8] + jne fail + mov rax, [output + 16] + or rax, [output + 24] + jne fail +%endmacro + +%macro TEST 1 + %assign mode (%1 & 3) + %if %1 & 4 + mov rdi, r8 + %else + lea rdi, [r8 + mode * 8] + %endif + vmovdqu ymm0, [rsi] + vmovdqu ymm1, [rsi] + vmovdqu ymm2, [rsi] + vroundsd xmm0, xmm1, xmm2, %1 + VERIFY ymm0, 0 + vmovdqu ymm1, [rsi] + vroundsd xmm1, xmm1, xmm2, %1 + VERIFY ymm1, 0 + vmovdqu ymm1, [rsi] + vroundsd xmm2, xmm1, xmm2, %1 + VERIFY ymm2, 0 + vmovdqu ymm1, [rsi] + vroundsd xmm0, xmm1, xmm1, %1 + VERIFY ymm0, 0 + vroundsd xmm1, xmm1, xmm1, %1 + VERIFY ymm1, 0 + vmovdqu ymm1, [rsi] + vroundsd xmm0, xmm1, [rsi], %1 + VERIFY ymm0, 0 + vroundsd xmm1, xmm1, [rsi], %1 + VERIFY ymm1, 0 +%endmacro + +section .text +_start: + xor r9d, r9d +.rounding: + mov eax, r9d + shl eax, 13 + or eax, 0x1f80 + mov [mxcsr], eax + ldmxcsr [mxcsr] + lea r10, [cases] + mov r11d, 10 +.case: + lea rsi, [initial] + mov rax, [r10] + mov [rsi], rax + lea r8, [r10 + 8] + %assign imm 0 + %rep 16 + %if imm & 4 + lea r8, [r10 + 8 + r9 * 8] + %else + lea r8, [r10 + 8] + %endif + TEST imm + %assign imm imm + 1 + %endrep + add r10, 40 + dec r11d + jnz .case + inc r9d + cmp r9d, 4 + jne .rounding + ; An enabled x87 invalid trap must keep the original rounding path. + fldcw [control] + lea rsi, [initial] + mov rax, [cases + 4 * 40] + mov [rsi], rax + lea rdi, [cases + 4 * 40 + 8 + 3 * 8] + vmovdqu ymm1, [rsi] + vmovdqu ymm2, [rsi] + vroundsd xmm0, xmm1, xmm2, 3 + VERIFY ymm0, 0 + fninit + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata +align 8 +cases: + ; Input, nearest-even, floor, ceil, truncation. + dq 0, 0, 0, 0, 0 + dq 0x8000000000000000, 0x8000000000000000, 0x8000000000000000, 0x8000000000000000, 0x8000000000000000 + dq 0x3fe0000000000000, 0, 0, 0x3ff0000000000000, 0 + dq 0xbfe0000000000000, 0x8000000000000000, 0xbff0000000000000, 0x8000000000000000, 0x8000000000000000 + dq 0x3ff8000000000000, 0x4000000000000000, 0x3ff0000000000000, 0x4000000000000000, 0x3ff0000000000000 + dq 0xbff8000000000000, 0xc000000000000000, 0xc000000000000000, 0xbff0000000000000, 0xbff0000000000000 + dq 0x4004000000000000, 0x4000000000000000, 0x4000000000000000, 0x4008000000000000, 0x4000000000000000 + dq 0xc004000000000000, 0xc000000000000000, 0xc008000000000000, 0xc000000000000000, 0xc000000000000000 + dq 0x432fffffffffffff, 0x4330000000000000, 0x432ffffffffffffe, 0x4330000000000000, 0x432ffffffffffffe + dq 0x4340000000000000, 0x4340000000000000, 0x4340000000000000, 0x4340000000000000, 0x4340000000000000 +control: dw 0x37e + +section .data +align 32 +initial: + dq 0, 0x123456789abcdef0, -1, -1 + +section .bss +align 32 +output: resb 32 +mxcsr: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits From a315b95f4818bbac70b0598fb8f01adfaf3a2fef Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Fri, 30 Oct 2026 18:29:26 +0800 Subject: [PATCH 10/33] LATX, opt: Avoid scalar-double AVX result copies When dest is src1 in VADDSD, VSUBSD, VMULSD or VDIVSD, insert the computed low double into dest. Its high double already contains the required src1 bits, so the temporary shuffle and full-register copy are unnecessary. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx.c | 32 +++++--- tests/integration/avx-scalar-double-alias.asm | 82 +++++++++++++++++++ .../integration/registrations/avx/meson.build | 1 + 3 files changed, 103 insertions(+), 12 deletions(-) create mode 100644 tests/integration/avx-scalar-double-alias.asm diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 78b6a5d4ce..cb929c655a 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -86,9 +86,11 @@ bool translate_vaddsd(IR1_INST * pir1) { ra_alloc_ftemp() : dest; la_fadd_d(result, src1, src2); - la_vshuf4i_d(result, src1, 0xc); - if (!ir2_opnd_cmp(&result, &dest)) - la_xvori_b(dest, result, 0); + if (ir2_opnd_cmp(&dest, &src1)) { + la_xvinsve0_d(dest, result, 0); + } else { + la_vshuf4i_d(dest, src1, 0xc); + } set_high128_xreg_to_zero(dest); return true; } @@ -191,9 +193,11 @@ bool translate_vsubsd(IR1_INST * pir1) { ra_alloc_ftemp() : dest; la_fsub_d(result, src1, src2); - la_vshuf4i_d(result, src1, 0xc); - if (!ir2_opnd_cmp(&result, &dest)) - la_xvori_b(dest, result, 0); + if (ir2_opnd_cmp(&dest, &src1)) { + la_xvinsve0_d(dest, result, 0); + } else { + la_vshuf4i_d(dest, src1, 0xc); + } set_high128_xreg_to_zero(dest); return true; } @@ -293,9 +297,11 @@ bool translate_vmulsd(IR1_INST * pir1) { ra_alloc_ftemp() : dest; la_fmul_d(result, src1, src2); - la_vshuf4i_d(result, src1, 0xc); - if (!ir2_opnd_cmp(&result, &dest)) - la_xvori_b(dest, result, 0); + if (ir2_opnd_cmp(&dest, &src1)) { + la_xvinsve0_d(dest, result, 0); + } else { + la_vshuf4i_d(dest, src1, 0xc); + } set_high128_xreg_to_zero(dest); return true; } @@ -395,9 +401,11 @@ bool translate_vdivsd(IR1_INST * pir1) { ra_alloc_ftemp() : dest; la_fdiv_d(result, src1, src2); - la_vshuf4i_d(result, src1, 0xc); - if (!ir2_opnd_cmp(&result, &dest)) - la_xvori_b(dest, result, 0); + if (ir2_opnd_cmp(&dest, &src1)) { + la_xvinsve0_d(dest, result, 0); + } else { + la_vshuf4i_d(dest, src1, 0xc); + } set_high128_xreg_to_zero(dest); return true; } diff --git a/tests/integration/avx-scalar-double-alias.asm b/tests/integration/avx-scalar-double-alias.asm new file mode 100644 index 0000000000..8c149fe5c6 --- /dev/null +++ b/tests/integration/avx-scalar-double-alias.asm @@ -0,0 +1,82 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +section .text + +%macro CHECK_RESULT 2 + mov edi, %2 + vmovdqu [actual], ymm0 + mov rax, [actual] + cmp rax, [%1] + jne fail + mov rax, [actual + 8] + cmp rax, [input_a + 8] + jne fail + mov rax, [actual + 16] + or rax, [actual + 24] + jne fail +%endmacro + +%macro CHECK_ALIASES 3 + vmovdqu ymm0, [junk] + vmovdqu ymm1, [input_a] + vmovdqu ymm2, [input_b] + %1 xmm0, xmm1, xmm2 + CHECK_RESULT %2, 1 + + vmovdqu ymm0, [input_a] + %1 xmm0, xmm0, xmm2 + CHECK_RESULT %2, 2 + + vmovdqu ymm0, [input_b] + %1 xmm0, xmm1, xmm0 + CHECK_RESULT %2, 3 + + vmovdqu ymm0, [junk] + %1 xmm0, xmm1, xmm1 + CHECK_RESULT %3, 4 + + vmovdqu ymm0, [input_a] + %1 xmm0, xmm0, xmm0 + CHECK_RESULT %3, 5 + + vmovdqu ymm0, [junk] + %1 xmm0, xmm1, [input_b] + CHECK_RESULT %2, 6 + + vmovdqu ymm0, [input_a] + %1 xmm0, xmm0, [input_b] + CHECK_RESULT %2, 7 +%endmacro + +_start: + mov edi, 1 + CHECK_ALIASES vaddsd, sum_ab, sum_aa + CHECK_ALIASES vsubsd, diff_ab, diff_aa + CHECK_ALIASES vmulsd, product_ab, product_aa + CHECK_ALIASES vdivsd, quotient_ab, quotient_aa + xor edi, edi +fail: + mov eax, 60 + syscall + +section .rodata align=32 +input_a: dq 6.0, 0x1122334455667788, -1, -1 +input_b: dq 2.0, 0x8877665544332211, -1, -1 +junk: times 4 dq 0xdeadbeefdeadbeef +sum_ab: dq 8.0 +sum_aa: dq 12.0 +diff_ab: dq 4.0 +diff_aa: dq 0.0 +product_ab: dq 12.0 +product_aa: dq 36.0 +quotient_ab: dq 3.0 +quotient_aa: dq 1.0 + +section .bss align=32 +actual: resb 32 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index cdd5aa0407..199b9a119a 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -104,6 +104,7 @@ if 'x86_64-linux-user' in target_dirs 'args': [ emulators['latx-x86_64'], files('../../avx-single-instruction-boundaries.asm'), + files('../../avx-scalar-double-alias.asm'), files('../../vblendpd-xmm-alias.asm'), files('../../avx128-shuffle.S'), files('../../avx-ymm-writeback.S'), From cb73354706e5cc66704ca6465b59f6bd41d381fd Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Fri, 30 Oct 2026 20:18:42 +0800 Subject: [PATCH 11/33] LATX, opt: Skip redundant FCSR masking in fixed VROUNDSS Check native exception enables before fixed-direction rounding. When all traps are masked, round without first disabling them in FCSR. With precision suppression, retain the unordered comparison and FCSR restore to preserve invalid status without adding inexact status. Otherwise, one round supplies both the result and exception status. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx.c | 35 +++++- .../integration/registrations/avx/meson.build | 1 + tests/integration/vroundss-fixed.asm | 118 ++++++++++++++++++ 3 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 tests/integration/vroundss-fixed.asm diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index cb929c655a..bd829a2f79 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -5551,6 +5551,38 @@ bool translate_vroundss(IR1_INST *pir1) IR2_OPND mxcsr = ra_alloc_itemp(); bool is_xmm = ir1_opnd_is_xmm(opnd0); la_xvreplve0_w(src, src2); + IR2_OPND merge = ra_alloc_label(); + if (!(imm & 0x4)) { + IR2_OPND slow = ra_alloc_label(); + + la_movfcsr2gr(fcsr, fcsr_ir2_opnd); + la_andi(fcsr_save, fcsr, FCSR_ENABLE_SET); + la_bnez(fcsr_save, slow); + if (imm & 0x8) { + /* Keep invalid status, but discard the round's inexact status. */ + la_vfcmp_cond_s(temp, src, src, 0x8); + la_movfcsr2gr(fcsr, fcsr_ir2_opnd); + } + switch (imm & 0x3) { + case 0: + la_vfrintrne_s(temp_dest, src); + break; + case 1: + la_vfrintrm_s(temp_dest, src); + break; + case 2: + la_vfrintrp_s(temp_dest, src); + break; + default: + la_vfrintrz_s(temp_dest, src); + break; + } + if (imm & 0x8) { + la_movgr2fcsr(fcsr_ir2_opnd, fcsr); + } + la_b(merge); + la_label(slow); + } if(imm & 0x8){ IR2_INST * ( * tr_inst)(IR2_OPND, IR2_OPND, IR2_OPND, int); tr_inst = is_xmm ? la_vfcmp_cond_s : la_xvfcmp_cond_s; @@ -5613,13 +5645,14 @@ bool translate_vroundss(IR1_INST *pir1) tr_inst = is_xmm ? la_vfrintrz_s : la_xvfrintrz_s; tr_inst(temp_dest, src); } + la_movgr2fcsr(fcsr_ir2_opnd, fcsr_save); + la_label(merge); if (ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) { la_xvori_b(dest, src1, 0); } la_xvinsve0_w(dest, temp_dest, 0); set_high128_xreg_to_zero(dest); - la_movgr2fcsr(fcsr_ir2_opnd, fcsr_save); ra_free_temp(temp); ra_free_temp(fcsr); diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index 199b9a119a..0c3692e61b 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -111,6 +111,7 @@ if 'x86_64-linux-user' in target_dirs files('../../vpshufd-distinct-regs.S'), files('../../vroundps-imm0b.S'), files('../../vroundsd-fixed.asm'), + files('../../vroundss-fixed.asm'), files('../../vcvtdq2pd-ymm-alias.asm'), files('../../avx-regression-extract-alias.S'), files('../../avx-regression-narrow-state.S'), diff --git a/tests/integration/vroundss-fixed.asm b/tests/integration/vroundss-fixed.asm new file mode 100644 index 0000000000..515aaa9ea2 --- /dev/null +++ b/tests/integration/vroundss-fixed.asm @@ -0,0 +1,118 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro VERIFY 1 + vmovdqu [output], %1 + mov eax, [output] + cmp eax, [rdi] + jne fail + mov eax, [output + 4] + cmp eax, [initial + 4] + jne fail + mov rax, [output + 8] + cmp rax, [initial + 8] + jne fail + mov rax, [output + 16] + or rax, [output + 24] + jne fail +%endmacro + +%macro TEST 1 + %if %1 & 4 + lea rdi, [r10 + 4 + r9 * 4] + %else + lea rdi, [r10 + 4 + (%1 & 3) * 4] + %endif + vmovdqu ymm0, [initial] + vmovdqu ymm1, [initial] + vmovdqu ymm2, [initial] + vroundss xmm0, xmm1, xmm2, %1 + VERIFY ymm0 + vmovdqu ymm1, [initial] + vroundss xmm1, xmm1, xmm2, %1 + VERIFY ymm1 + vmovdqu ymm1, [initial] + vroundss xmm2, xmm1, xmm2, %1 + VERIFY ymm2 + vmovdqu ymm1, [initial] + vroundss xmm0, xmm1, xmm1, %1 + VERIFY ymm0 + vroundss xmm1, xmm1, xmm1, %1 + VERIFY ymm1 + vmovdqu ymm1, [initial] + vroundss xmm0, xmm1, [initial], %1 + VERIFY ymm0 + vroundss xmm1, xmm1, [initial], %1 + VERIFY ymm1 +%endmacro + +section .text +_start: + xor r9d, r9d +.rounding: + mov eax, r9d + shl eax, 13 + or eax, 0x1f80 + mov [mxcsr], eax + ldmxcsr [mxcsr] + lea r10, [cases] + mov r11d, 10 +.case: + mov eax, [r10] + mov [initial], eax + %assign imm 0 + %rep 16 + TEST imm + %assign imm imm + 1 + %endrep + add r10, 20 + dec r11d + jnz .case + inc r9d + cmp r9d, 4 + jne .rounding + ; Enabled x87 invalid must retain the original fixed-round path. + fldcw [control] + lea r10, [cases + 4 * 20] + mov eax, [r10] + mov [initial], eax + TEST 3 + TEST 9 + fninit + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=4 +cases: + ; Input, nearest-even, floor, ceil, truncation. + dd 0, 0, 0, 0, 0 + dd 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000 + dd 0x3f000000, 0, 0, 0x3f800000, 0 + dd 0xbf000000, 0x80000000, 0xbf800000, 0x80000000, 0x80000000 + dd 0x3fc00000, 0x40000000, 0x3f800000, 0x40000000, 0x3f800000 + dd 0xbfc00000, 0xc0000000, 0xc0000000, 0xbf800000, 0xbf800000 + dd 0x40200000, 0x40000000, 0x40000000, 0x40400000, 0x40000000 + dd 0xc0200000, 0xc0000000, 0xc0400000, 0xc0000000, 0xc0000000 + dd 0x4affffff, 0x4b000000, 0x4afffffe, 0x4b000000, 0x4afffffe + dd 0x4b800000, 0x4b800000, 0x4b800000, 0x4b800000, 0x4b800000 +control: dw 0x37e + +section .data align=32 +initial: + dd 0, 0x89abcdef + dq 0x123456789abcdef0, -1, -1 + +section .bss align=32 +output: resb 32 +mxcsr: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits From 38bdbb05b97bba8b1d02c04be9d6f59ca5f39934 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Fri, 30 Oct 2026 21:46:31 +0800 Subject: [PATCH 12/33] LATX, opt: Reduce XMM VADDSUBPD lane preparation Compute the low subtraction and high addition with scalar operations, then interleave their results. This replaces seven core instructions with five and avoids arithmetic on unused elements. Read both sources before writing dest so overlapping registers remain valid. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx.c | 25 +++--- .../integration/registrations/avx/meson.build | 1 + tests/integration/vaddsubpd-lanes.asm | 86 +++++++++++++++++++ 3 files changed, 102 insertions(+), 10 deletions(-) create mode 100644 tests/integration/vaddsubpd-lanes.asm diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index bd829a2f79..095dac5420 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -541,6 +541,19 @@ bool translate_vaddsubpd(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(opnd2); + if (ir1_opnd_is_xmm(opnd0)) { + IR2_OPND low = ra_alloc_ftemp(); + IR2_OPND high1 = ra_alloc_ftemp(); + IR2_OPND high2 = ra_alloc_ftemp(); + + la_fsub_d(low, src1, src2); + la_xvpickve_d(high1, src1, 1); + la_xvpickve_d(high2, src2, 1); + la_fadd_d(high1, high1, high2); + la_vilvl_d(dest, high1, low); + set_high128_xreg_to_zero(dest); + return true; + } IR2_OPND add_src1 = ra_alloc_ftemp(); IR2_OPND add_src2 = ra_alloc_ftemp(); IR2_OPND sub_src1 = ra_alloc_ftemp(); @@ -550,17 +563,9 @@ bool translate_vaddsubpd(IR1_INST * pir1) { la_xvpackev_d(sub_src2, src2, src2); la_xvpackod_d(add_src1, src1, src1); la_xvpackod_d(add_src2, src2, src2); - if (ir1_opnd_is_xmm(opnd0)) { - la_vfsub_d(sub_src1, sub_src1, sub_src2); - la_vfadd_d(add_src1, add_src1, add_src2); - } else { - la_xvfsub_d(sub_src1, sub_src1, sub_src2); - la_xvfadd_d(add_src1, add_src1, add_src2); - } + la_xvfsub_d(sub_src1, sub_src1, sub_src2); + la_xvfadd_d(add_src1, add_src1, add_src2); la_xvpackev_d(dest, add_src1, sub_src1); - if (ir1_opnd_is_xmm(opnd0)) { - set_high128_xreg_to_zero(dest); - } return true; } diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index 0c3692e61b..e8a9c07504 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -105,6 +105,7 @@ if 'x86_64-linux-user' in target_dirs emulators['latx-x86_64'], files('../../avx-single-instruction-boundaries.asm'), files('../../avx-scalar-double-alias.asm'), + files('../../vaddsubpd-lanes.asm'), files('../../vblendpd-xmm-alias.asm'), files('../../avx128-shuffle.S'), files('../../avx-ymm-writeback.S'), diff --git a/tests/integration/vaddsubpd-lanes.asm b/tests/integration/vaddsubpd-lanes.asm new file mode 100644 index 0000000000..c4d8fcbb5b --- /dev/null +++ b/tests/integration/vaddsubpd-lanes.asm @@ -0,0 +1,86 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +section .text + +%macro CHECK 2 + mov edi, %2 + vmovdqu [actual], ymm0 + mov rax, [actual] + cmp rax, [%1] + jne fail + mov rax, [actual + 8] + cmp rax, [%1 + 8] + jne fail + mov rax, [actual + 16] + or rax, [actual + 24] + jne fail +%endmacro + +%macro ALIASES 0 + vmovdqu ymm0, [junk] + vmovdqu ymm1, [input_a] + vmovdqu ymm2, [input_b] + vaddsubpd xmm0, xmm1, xmm2 + CHECK expected_ab, 1 + vmovdqu ymm0, [input_a] + vaddsubpd xmm0, xmm0, xmm2 + CHECK expected_ab, 2 + vmovdqu ymm0, [input_b] + vaddsubpd xmm0, xmm1, xmm0 + CHECK expected_ab, 3 + vaddsubpd xmm0, xmm1, xmm1 + CHECK rbx, 4 + vmovdqu ymm0, [input_a] + vaddsubpd xmm0, xmm0, xmm0 + CHECK rbx, 5 + vaddsubpd xmm0, xmm1, [input_b] + CHECK expected_ab, 6 + vmovdqu ymm0, [input_a] + vaddsubpd xmm0, xmm0, [input_b] + CHECK expected_ab, 7 +%endmacro + +_start: + mov ecx, 4 +.rounding: + lea rbx, [expected_aa] + cmp ecx, 2 + jne .expected_ready + lea rbx, [expected_aa_down] +.expected_ready: + mov eax, ecx + dec eax + shl eax, 13 + or eax, 0x1f80 + mov [mxcsr], eax + ldmxcsr [mxcsr] + ALIASES + ; Unused YMM lanes contain signaling NaNs and must not raise invalid. + stmxcsr [mxcsr] + mov edi, 8 + test dword [mxcsr], 1 + jnz fail + dec ecx + jnz .rounding + xor edi, edi +fail: + mov eax, 60 + syscall + +section .rodata align=32 +input_a: dq 6.0, -4.0, 0x7ff0000000000001, 0x7ff0000000000001 +input_b: dq 2.0, 1.0, 0x7ff0000000000001, 0x7ff0000000000001 +junk: times 4 dq 0xdeadbeefdeadbeef +expected_ab: dq 4.0, -3.0 +expected_aa: dq 0.0, -8.0 +expected_aa_down: dq 0x8000000000000000, -8.0 + +section .bss align=32 +actual: resb 32 +mxcsr: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits From 3663fdb5a6904078a2cee707a7b45e4d53b8e602 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Sat, 31 Oct 2026 00:29:19 +0800 Subject: [PATCH 13/33] LATX, opt: Skip VROUNDSS FCSR restore when inexact is already set For fixed rounds with precision suppression, skip the FCSR save/restore when native traps are masked and inexact is already set. Round once, then perform the unordered comparison to produce the final cause bits. Use the existing sequence when inexact is clear, traps are enabled or MXCSR selects the rounding direction. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx.c | 38 +++++++++++++++++++--------- tests/integration/vroundss-fixed.asm | 33 ++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 095dac5420..0bf57f51dc 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -5559,33 +5559,47 @@ bool translate_vroundss(IR1_INST *pir1) IR2_OPND merge = ra_alloc_label(); if (!(imm & 0x4)) { IR2_OPND slow = ra_alloc_label(); + IR2_OPND sticky = ra_alloc_label(); + IR2_INST *(*fixed_round)(IR2_OPND, IR2_OPND); - la_movfcsr2gr(fcsr, fcsr_ir2_opnd); - la_andi(fcsr_save, fcsr, FCSR_ENABLE_SET); - la_bnez(fcsr_save, slow); - if (imm & 0x8) { - /* Keep invalid status, but discard the round's inexact status. */ - la_vfcmp_cond_s(temp, src, src, 0x8); - la_movfcsr2gr(fcsr, fcsr_ir2_opnd); - } switch (imm & 0x3) { case 0: - la_vfrintrne_s(temp_dest, src); + fixed_round = la_vfrintrne_s; break; case 1: - la_vfrintrm_s(temp_dest, src); + fixed_round = la_vfrintrm_s; break; case 2: - la_vfrintrp_s(temp_dest, src); + fixed_round = la_vfrintrp_s; break; default: - la_vfrintrz_s(temp_dest, src); + fixed_round = la_vfrintrz_s; break; } + + la_movfcsr2gr(fcsr, fcsr_ir2_opnd); + la_andi(fcsr_save, fcsr, FCSR_ENABLE_SET); + la_bnez(fcsr_save, slow); + if (imm & 0x8) { + la_bstrpick_w(fcsr_save, fcsr, FCSR_OFF_FLAGS_I, + FCSR_OFF_FLAGS_I); + la_bnez(fcsr_save, sticky); + /* Keep invalid status, but discard the round's inexact status. */ + la_vfcmp_cond_s(temp, src, src, 0x8); + la_movfcsr2gr(fcsr, fcsr_ir2_opnd); + } + fixed_round(temp_dest, src); if (imm & 0x8) { la_movgr2fcsr(fcsr_ir2_opnd, fcsr); } la_b(merge); + if (imm & 0x8) { + la_label(sticky); + /* Inexact is already sticky; the compare restores final causes. */ + fixed_round(temp_dest, src); + la_vfcmp_cond_s(temp, src, src, 0x8); + la_b(merge); + } la_label(slow); } if(imm & 0x8){ diff --git a/tests/integration/vroundss-fixed.asm b/tests/integration/vroundss-fixed.asm index 515aaa9ea2..dab79e01a4 100644 --- a/tests/integration/vroundss-fixed.asm +++ b/tests/integration/vroundss-fixed.asm @@ -75,6 +75,37 @@ _start: inc r9d cmp r9d, 4 jne .rounding + ; A preexisting inexact flag must survive precision-suppressed rounds. + xor r9d, r9d +.sticky: + mov eax, r9d + shl eax, 13 + or eax, 0x1f80 + mov [mxcsr], eax + ldmxcsr [mxcsr] + lea r10, [cases + 4 * 20] + mov eax, [r10] + mov [initial], eax + fnclex + vmovss xmm15, [initial] + vroundss xmm15, xmm15, xmm15, 3 + fnstsw [status] + stmxcsr [mxcsr_before] + TEST 8 + TEST 9 + TEST 10 + TEST 11 + fnstsw ax + xor ax, [status] + test ax, 0x3f + jnz fail + stmxcsr [mxcsr] + mov eax, [mxcsr] + cmp eax, [mxcsr_before] + jne fail + inc r9d + cmp r9d, 4 + jne .sticky ; Enabled x87 invalid must retain the original fixed-round path. fldcw [control] lea r10, [cases + 4 * 20] @@ -114,5 +145,7 @@ initial: section .bss align=32 output: resb 32 mxcsr: resd 1 +mxcsr_before: resd 1 +status: resw 1 section .note.GNU-stack noalloc noexec nowrite progbits From 358e14e3c332955f64ae0a2bf940015555c8109c Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Sat, 31 Oct 2026 17:28:52 +0800 Subject: [PATCH 14/33] LATX, opt: Fold large base displacements into load/store offsets For eligible 64-bit base-plus-displacement addresses, add the rounded high part to the base and keep the signed low 12 bits in the memory instruction. This removes an ORI from address generation when the high part fits signed32. Exclude indexed, segmented, RIP-relative and narrow-address forms. Keep 80-bit x87 operands on the old path because their second access adds eight to the returned displacement. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/mem-interface.c | 18 ++ tests/integration/base-displacement.asm | 260 ++++++++++++++++++ .../integration/registrations/avx/meson.build | 8 + 3 files changed, 286 insertions(+) create mode 100644 tests/integration/base-displacement.asm diff --git a/target/i386/latx/translator/mem-interface.c b/target/i386/latx/translator/mem-interface.c index a1b6731759..9566494649 100644 --- a/target/i386/latx/translator/mem-interface.c +++ b/target/i386/latx/translator/mem-interface.c @@ -367,6 +367,24 @@ static IR2_OPND convert_mem_helper(IR1_OPND *opnd1, IR2_OPND *arg_dest_op, if (host_off) { if (si12_overflow(offset)) { *host_off = 0; +#ifdef TARGET_X86_64 + bool split_offset = CODEIS64 && ir1_addr_size(pir1) == 64 && + has_base && !has_index && !has_seg && + ir1_opnd_size(opnd1) != 80; +#ifdef CONFIG_LATX_IMM_REG + split_offset = split_offset && + (!option_imm_reg || !option_imm_complex); +#endif + if (split_offset) { + longx low = sextract64(offset, 0, 12); + longx high = offset - low; + if (low && high == (int32_t)high) { + /* Keep the low displacement in the load/store itself. */ + *host_off = low; + offset = high; + } + } +#endif } else { #ifdef CONFIG_LATX_IMM_REG cache_skip = true; diff --git a/tests/integration/base-displacement.asm b/tests/integration/base-displacement.asm new file mode 100644 index 0000000000..d0bcdd220a --- /dev/null +++ b/tests/integration/base-displacement.asm @@ -0,0 +1,260 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +%macro CHECK_OFFSET 1 + lea rbx, [data] + mov r14, rbx + mov rax, %1 + sub rbx, rax + mov rax, [rbx + %1] + cmp rax, [data] + jne fail + mov rax, 0x8877665544332211 + mov [rbx + %1], rax + cmp [data], rax + jne fail + lea rax, [rbx + %1] + cmp rax, r14 + jne fail + mov rax, [rbx + %1 + 8] + cmp rax, [data + 8] + jne fail + vmovdqu ymm0, [vector] + vmovdqu [rbx + %1], ymm0 + vmovdqu ymm1, [rbx + %1] + vpxor ymm1, ymm1, ymm0 + vptest ymm1, ymm1 + jnz fail + mov rax, [data] + mov rcx, 5 + add qword [rbx + %1], rcx + add rax, rcx + cmp [data], rax + jne fail + xchg rax, [rbx + %1] + cmp [data], rax + jne fail + mov ecx, 1 + bt qword [rbx + %1], rcx + setc dl + bt qword [data], rcx + setc cl + cmp dl, cl + jne fail +%endmacro + +section .text +_start: + CHECK_OFFSET -2147483648 + CHECK_OFFSET -65537 + CHECK_OFFSET -29904 + CHECK_OFFSET -29124 + CHECK_OFFSET -4097 + CHECK_OFFSET -2049 + CHECK_OFFSET -2048 + CHECK_OFFSET 0 + CHECK_OFFSET 2047 + CHECK_OFFSET 2048 + CHECK_OFFSET 4095 + CHECK_OFFSET 4096 + CHECK_OFFSET 2147481599 + CHECK_OFFSET 2147481600 + CHECK_OFFSET 2147483632 + + ; An aliased load must consume the base before publishing its result. + lea rax, [data - 29904] + mov rax, [rax + 29904] + cmp rax, [data] + jne fail + lea rax, [data + 29904] + mov eax, [rax - 29904] + cmp eax, [data] + jne fail + lea rbp, [data - 29904] + mov rbp, [rbp + 29904] + cmp rbp, [data] + jne fail + mov r15, rsp + lea rsp, [data + 29904] + mov rsp, [rsp - 29904] + mov rax, rsp + mov rsp, r15 + cmp rax, [data] + jne fail + lea rbx, [data - 29904] + mov [rbx + 29904], rbx + cmp [data], rbx + jne fail + + ; Register bit offsets may move the access away from the nominal base. + mov qword [data], 1 + mov qword [data + 8], 0 + mov ecx, 64 + bts qword [rbx + 29904], rcx + jc fail + cmp qword [data + 8], 1 + jne fail + mov rcx, -64 + btr qword [rbx + 29912], rcx + jnc fail + cmp qword [data], 0 + jne fail + + ; Atomic and no-offset callers retain complete addresses. + mov qword [data], 7 + mov eax, 7 + mov edx, 11 + lock cmpxchg [rbx + 29904], rdx + jne fail + cmp qword [data], 11 + jne fail + mov edx, 5 + lock xadd [rbx + 29904], rdx + cmp edx, 11 + jne fail + cmp qword [data], 16 + jne fail + + ; Legacy SSE shares the helper. x87 needs room for a second access at +8. + movdqu xmm0, [vector] + movdqu [rbx + 29904], xmm0 + movdqu xmm1, [rbx + 29904] + pxor xmm1, xmm0 + ptest xmm1, xmm1 + jnz fail + lea rbx, [data - 6143] + fld tword [one] + fstp tword [rbx + 6143] + fld tword [rbx + 6143] + fstp tword [scratch] + mov rax, [scratch] + cmp rax, [one] + jne fail + mov ax, [scratch + 8] + cmp ax, [one + 8] + jne fail + + ; Address-size, index and segment fallbacks must still include every term. + lea rbx, [data] + sub rbx, 29904 + mov rax, [ebx + 29904] + cmp rax, [data] + jne fail + mov ecx, 7 + sub rbx, rcx + mov rax, [rbx + rcx + 29904] + cmp rax, [data] + jne fail + mov eax, 158 + mov edi, 0x1002 + lea rsi, [data] + syscall + test eax, eax + jnz fail + mov rbx, 29904 + mov rax, [fs:rbx - 29904] + cmp rax, [data] + jne fail + mov eax, 158 + mov edi, 0x1002 + xor esi, esi + syscall + + ; Fault on each crossing access, reopen the protected region, then retry. + mov eax, 13 + mov edi, 11 + lea rsi, [action] + xor edx, edx + mov r10d, 8 + syscall + test eax, eax + jnz fail + mov eax, 9 + xor edi, edi + mov esi, 196608 + mov edx, 3 + mov r10d, 0x22 + mov r8, -1 + xor r9d, r9d + syscall + cmp rax, -4095 + jae fail + mov r12, rax + lea r13, [r12 + 65536] + lea rbx, [r13 - 16 + 29904] + vmovdqu ymm0, [vector] + vmovdqu [rbx - 29904], ymm0 + call protect + lea rax, [fault_load] + mov [expected_pc], rax +fault_load: + vmovdqu ymm1, [rbx - 29904] + vpxor ymm2, ymm1, ymm0 + vptest ymm2, ymm2 + jnz fail + call protect + lea rax, [fault_store] + mov [expected_pc], rax +fault_store: + vmovdqu [rbx - 29904], ymm0 + cmp dword [faults], 2 + jne fail + vmovdqu ymm2, [rbx - 29904] + vpxor ymm2, ymm2, ymm0 + vptest ymm2, ymm2 + jnz fail + xor edi, edi + jmp exit +protect: + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + xor edx, edx + syscall + test eax, eax + jnz fail + ret +handler: + cmp edi, 11 + jne fail + cmp [rsi + 16], r13 + jne fail + mov rax, [expected_pc] + cmp [rdx + 168], rax + jne fail + inc dword [faults] + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + mov edx, 3 + syscall + test eax, eax + jnz fail + ret +restorer: + mov eax, 15 + syscall + ud2 +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +vector: dq 0x8877665544332211, 0x1122334455667788 + dq 0x0123456789abcdef, 0xfedcba9876543210 +one: dq 0x8000000000000000 + dw 0x3fff +action: dq handler, 0x04000004, restorer, 0 +section .data align=32 +data: times 4 dq 0x8877665544332211 +section .bss align=8 +faults: resd 1 +alignb 8 +expected_pc: resq 1 +scratch: resb 16 +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index e8a9c07504..0e11dc576e 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -133,6 +133,14 @@ if 'x86_64-linux-user' in target_dirs suite: 'latx-integration', ) foreach cache_option : ['0000', '1100', '1101', '1110', '1111'] + latx_integration_tests += [{ + 'name': 'test-base-displacement-' + cache_option, + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], + files('../../base-displacement.asm')], + 'env': {'LATX_IMM_REG': cache_option, 'LATX_SOFTFPU': '0'}, + 'timeout': 120, + }] test( 'test-rip-relative-cache-' + cache_option, find_program('../../test-avx-regressions.py'), From ef2a5a08c9a3bc8ef8a8594c5e1d79363d4d4569 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Sat, 31 Oct 2026 20:51:41 +0800 Subject: [PATCH 15/33] LATX, fix: Recover delayed flags without translation-time IR1 data Signal recovery reads tb->s_data->ir1 even though recovered TBs may have no IR1 list or share storage with another TB. A guest fault can then cause a nested host fault or report the wrong guest address. Store the delayed CMP/TEST/BT operation, width and TEST alias information in the per-instruction search table. Recover flags from saved host operands without reading temporary translation data, and activate the recovery record only after the comparison completes. Signed-off-by: Zhaoyang Xie --- accel/tcg/translate-all.c | 4 - target/i386/latx/include/insts-pattern.h | 5 + target/i386/latx/include/ir2.h | 3 - target/i386/latx/include/latx-unwind.h | 29 ++ target/i386/latx/translator/tr-pattern.c | 382 +++++++++--------- target/i386/latx/translator/translate.c | 27 +- target/i386/tcg/translate.c | 12 +- .../instptn-signal-recovery-i386.S | 225 +++++++++++ tests/integration/instptn-signal-recovery.S | 268 ++++++++++++ .../integration/registrations/avx/meson.build | 16 + 10 files changed, 773 insertions(+), 198 deletions(-) create mode 100644 target/i386/latx/include/latx-unwind.h create mode 100644 tests/integration/instptn-signal-recovery-i386.S create mode 100644 tests/integration/instptn-signal-recovery.S diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index 2ac697c8bb..06d1a55268 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -424,7 +424,6 @@ int encode_search(TranslationBlock *tb, uint8_t *block) #ifdef CONFIG_LATX_OPT_PUSH_POP extern void ir2_opt_push_pop_fix(TranslationBlock *tb, CPUState *cpu, int i); #endif -extern void opt_instptn_fix(CPUState *cpu, TranslationBlock *tb, int i); /* The cpu state corresponding to 'searched_pc' is restored. * When reset_icount is true, current TB will be interrupted and @@ -486,9 +485,6 @@ static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, ir2_opt_push_pop_fix(tb, cpu, (searched_pc - (uintptr_t)tb->tc.ptr) / 4); } #endif -#ifdef CONFIG_LATX_INSTS_PATTERN - opt_instptn_fix(cpu, tb, i); -#endif #ifdef CONFIG_PROFILER qatomic_set(&prof->restore_time, prof->restore_time + profile_getclock() - ti); diff --git a/target/i386/latx/include/insts-pattern.h b/target/i386/latx/include/insts-pattern.h index 781a6677ca..33dc3d8ae4 100644 --- a/target/i386/latx/include/insts-pattern.h +++ b/target/i386/latx/include/insts-pattern.h @@ -163,4 +163,9 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *ir1, int index, bool try_translate_instptn(IR1_INST *pir1); +#ifdef CONFIG_LATX_INSTS_PATTERN +unsigned instptn_flags_recovery(IR1_INST *pir1); +void opt_instptn_fix(CPUArchState *env, unsigned recovery); +#endif + #endif diff --git a/target/i386/latx/include/ir2.h b/target/i386/latx/include/ir2.h index 208d64cfb2..b052475f97 100644 --- a/target/i386/latx/include/ir2.h +++ b/target/i386/latx/include/ir2.h @@ -116,8 +116,5 @@ bool la_ir2_opcode_is_load(IR2_OPCODE opcode); void tr_ir2_optimize(TranslationBlock *tb); void ir2_opt_push_pop_fix(TranslationBlock *tb, CPUState *cpu, int i); -#ifdef CONFIG_LATX_INSTS_PATTERN -void opt_instptn_fix(CPUState *cpu, TranslationBlock *tb, int index); -#endif void la_append_ir2_jmp_far(ADDR jmp_offset); #endif diff --git a/target/i386/latx/include/latx-unwind.h b/target/i386/latx/include/latx-unwind.h new file mode 100644 index 0000000000..baf9dbb5b1 --- /dev/null +++ b/target/i386/latx/include/latx-unwind.h @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: 2021-2026 LAT Project Authors + * + * SPDX-License-Identifier: GPL-2.0-only + */ + +#ifndef LATX_UNWIND_H +#define LATX_UNWIND_H + +/* Layout of the second per-instruction search-table word, also on i386. */ +#define LATX_UNWIND_CC_OP_MASK 0xff +#define LATX_UNWIND_INSTPTN_SHIFT 8 +#define LATX_UNWIND_INSTPTN_MASK 0xff +#define LATX_UNWIND_YMMH_SHIFT 16 + +/* Delayed integer flags use the saved host a0/a1, never temporary IR1. */ +enum { + INSTPTN_FLAGS_NONE, + INSTPTN_FLAGS_CMP, + INSTPTN_FLAGS_TEST, + INSTPTN_FLAGS_BT, +}; + +#define INSTPTN_FLAGS_OP_MASK 3 +#define INSTPTN_FLAGS_SIZE_SHIFT 2 +#define INSTPTN_FLAGS_SIZE_MASK 3 +#define INSTPTN_FLAGS_SAME_REG (1 << 4) + +#endif diff --git a/target/i386/latx/translator/tr-pattern.c b/target/i386/latx/translator/tr-pattern.c index 70d85d9bf1..9fc717eb45 100644 --- a/target/i386/latx/translator/tr-pattern.c +++ b/target/i386/latx/translator/tr-pattern.c @@ -12,6 +12,7 @@ #include "insts-pattern.h" #include "tu.h" #include "hbr.h" +#include "latx-unwind.h" #ifdef CONFIG_LATX_INSTS_PATTERN @@ -2562,195 +2563,206 @@ bool try_translate_instptn(IR1_INST *pir1) return false; } -void opt_instptn_fix(CPUState *cpu, TranslationBlock *tb, int index) +unsigned instptn_flags_recovery(IR1_INST *pir1) { - CPUArchState *env = cpu->env_ptr; - // int num_insns = tb->icount; - for (int i = 0; i < index; i++) { - IR1_INST *pir1 = tb_ir1_inst(tb, i); - if (pir1->instptn.opc == INSTPTN_OPC_CMP_XX_JCC || - pir1->instptn.opc == INSTPTN_OPC_TEST_XX_JCC || - pir1->instptn.opc == INSTPTN_OPC_BT_XX_JCC) { - - int opnd_size = ir1_opnd_size(ir1_get_opnd(pir1, 0)); - ucontext_t *uc = env->puc; - - switch (ir1_opcode(pir1)) - { - case WRAP(CMP): - { - uint64_t a0 = UC_GR(uc)[4]; - uint64_t a1 = UC_GR(uc)[5]; - uint64_t eflags = 0; - switch (opnd_size) - { - case 8: - asm volatile ( - DEFINE_PARSE_R - "parse_r_%= __src, %[src] \n\t" - "parse_r_%= __dst, %[dst] \n\t" - ".word (0x003f0008 | (__src << 10) | (__dst << 5))\n\t" /*x86sub.b*/ - "parse_r_%= __flags, %[flags] \n\t" - ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ - : [dst]"+r"(a0), - [flags]"=r"(eflags) - : [src]"r"(a1) - : "cc" - ); - break; - case 16: - asm volatile ( - DEFINE_PARSE_R - "parse_r_%= __src, %[src] \n\t" - "parse_r_%= __dst, %[dst] \n\t" - ".word (0x003f0009 | (__src << 10) | (__dst << 5))\n\t" /*x86sub.h*/ - "parse_r_%= __flags, %[flags] \n\t" - ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ - : [dst]"+r"(a0), - [flags]"=r"(eflags) - : [src]"r"(a1) - : "cc" - ); - break; - case 32: - asm volatile ( - DEFINE_PARSE_R - "parse_r_%= __src, %[src] \n\t" - "parse_r_%= __dst, %[dst] \n\t" - ".word (0x003f000a | (__src << 10) | (__dst << 5))\n\t" /*x86sub.w*/ - "parse_r_%= __flags, %[flags] \n\t" - ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ - : [dst]"+r"(a0), - [flags]"=r"(eflags) - : [src]"r"(a1) - : "cc" - ); - break; - case 64: - asm volatile ( - DEFINE_PARSE_R - "parse_r_%= __src, %[src] \n\t" - "parse_r_%= __dst, %[dst] \n\t" - ".word (0x003f000b | (__src << 10) | (__dst << 5))\n\t" /*x86sub.d*/ - "parse_r_%= __flags, %[flags] \n\t" - ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ - : [dst]"+r"(a0), - [flags]"=r"(eflags) - : [src]"r"(a1) - : "cc" - ); - break; - default: - break; - } - env->eflags = env->eflags & ~(0b100011010101); - env->eflags = env->eflags | (eflags & 0b100011010101); - } - break; - case WRAP(TEST): - { - IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0); - IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); - - bool is_same_reg = ir1_opnd_is_same_reg(opnd0, opnd1); - uint64_t a0 = UC_GR(uc)[4]; - uint64_t a1 = is_same_reg ? a0 : UC_GR(uc)[5]; - uint64_t eflags = 0; - - switch (opnd_size) - { - case 8: - asm volatile ( - DEFINE_PARSE_R - "parse_r_%= __src, %[src] \n\t" - "parse_r_%= __dst, %[dst] \n\t" - ".word (0x003f8010 | (__src << 10) | (__dst << 5))\n\t" /*x86and.b*/ - "parse_r_%= __flags, %[flags] \n\t" - ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ - : [dst]"+r"(a0), - [flags]"=r"(eflags) - : [src]"r"(a1) - : "cc" - ); - break; - case 16: - asm volatile ( - DEFINE_PARSE_R - "parse_r_%= __src, %[src] \n\t" - "parse_r_%= __dst, %[dst] \n\t" - ".word (0x003f8011 | (__src << 10) | (__dst << 5))\n\t" /*x86and.h*/ - "parse_r_%= __flags, %[flags] \n\t" - ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ - : [dst]"+r"(a0), - [flags]"=r"(eflags) - : [src]"r"(a1) - : "cc" - ); - break; - case 32: - asm volatile ( - DEFINE_PARSE_R - "parse_r_%= __src, %[src] \n\t" - "parse_r_%= __dst, %[dst] \n\t" - ".word (0x003f8012 | (__src << 10) | (__dst << 5))\n\t" /*x86and.w*/ - "parse_r_%= __flags, %[flags] \n\t" - ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ - : [dst]"+r"(a0), - [flags]"=r"(eflags) - : [src]"r"(a1) - : "cc" - ); - break; - case 64: - asm volatile ( - DEFINE_PARSE_R - "parse_r_%= __src, %[src] \n\t" - "parse_r_%= __dst, %[dst] \n\t" - ".word (0x003f8013 | (__src << 10) | (__dst << 5))\n\t" /*x86and.d*/ - "parse_r_%= __flags, %[flags] \n\t" - ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ - : [dst]"+r"(a0), - [flags]"=r"(eflags) - : [src]"r"(a1) - : "cc" - ); - break; - default: - break; - } - env->eflags = env->eflags & ~(0b100011000101); - env->eflags = env->eflags | (eflags & 0b100011000101); - } - break; - case WRAP(BT): - { - uint64_t a0 = UC_GR(uc)[4]; - uint64_t a1 = UC_GR(uc)[5]; - env->eflags = env->eflags & ~(1); // clear CF - env->eflags = env->eflags | ((a0 >> a1) & 1); // set CF - } + unsigned recovery; + + switch (pir1->instptn.opc) { + case INSTPTN_OPC_CMP_XX_JCC: + if (ir1_opcode(pir1) != WRAP(CMP)) { + return INSTPTN_FLAGS_NONE; + } + recovery = INSTPTN_FLAGS_CMP; + break; + case INSTPTN_OPC_TEST_XX_JCC: + if (ir1_opcode(pir1) != WRAP(TEST)) { + return INSTPTN_FLAGS_NONE; + } + recovery = INSTPTN_FLAGS_TEST; + if (ir1_opnd_is_same_reg(ir1_get_opnd(pir1, 0), + ir1_get_opnd(pir1, 1))) { + recovery |= INSTPTN_FLAGS_SAME_REG; + } + break; + case INSTPTN_OPC_BT_XX_JCC: + if (ir1_opcode(pir1) != WRAP(BT)) { + return INSTPTN_FLAGS_NONE; + } + recovery = INSTPTN_FLAGS_BT; + break; + default: + return INSTPTN_FLAGS_NONE; + } + + int size = ir1_opnd_size(ir1_get_opnd(pir1, 0)); + + lsassert(size == 8 || size == 16 || size == 32 || size == 64); + return recovery | ((__builtin_ctz(size) - 3) << INSTPTN_FLAGS_SIZE_SHIFT); +} + +void opt_instptn_fix(CPUArchState *env, unsigned recovery) +{ + int opnd_size = 8 << ((recovery >> INSTPTN_FLAGS_SIZE_SHIFT) & + INSTPTN_FLAGS_SIZE_MASK); + ucontext_t *uc = env->puc; + + switch (recovery & INSTPTN_FLAGS_OP_MASK) + { + case INSTPTN_FLAGS_CMP: + { + uint64_t a0 = UC_GR(uc)[4]; + uint64_t a1 = UC_GR(uc)[5]; + uint64_t eflags = 0; + switch (opnd_size) + { + case 8: + asm volatile ( + DEFINE_PARSE_R + "parse_r_%= __src, %[src] \n\t" + "parse_r_%= __dst, %[dst] \n\t" + ".word (0x003f0008 | (__src << 10) | (__dst << 5))\n\t" /*x86sub.b*/ + "parse_r_%= __flags, %[flags] \n\t" + ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ + : [dst]"+r"(a0), + [flags]"=r"(eflags) + : [src]"r"(a1) + : "cc" + ); + break; + case 16: + asm volatile ( + DEFINE_PARSE_R + "parse_r_%= __src, %[src] \n\t" + "parse_r_%= __dst, %[dst] \n\t" + ".word (0x003f0009 | (__src << 10) | (__dst << 5))\n\t" /*x86sub.h*/ + "parse_r_%= __flags, %[flags] \n\t" + ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ + : [dst]"+r"(a0), + [flags]"=r"(eflags) + : [src]"r"(a1) + : "cc" + ); + break; + case 32: + asm volatile ( + DEFINE_PARSE_R + "parse_r_%= __src, %[src] \n\t" + "parse_r_%= __dst, %[dst] \n\t" + ".word (0x003f000a | (__src << 10) | (__dst << 5))\n\t" /*x86sub.w*/ + "parse_r_%= __flags, %[flags] \n\t" + ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ + : [dst]"+r"(a0), + [flags]"=r"(eflags) + : [src]"r"(a1) + : "cc" + ); + break; + case 64: + asm volatile ( + DEFINE_PARSE_R + "parse_r_%= __src, %[src] \n\t" + "parse_r_%= __dst, %[dst] \n\t" + ".word (0x003f000b | (__src << 10) | (__dst << 5))\n\t" /*x86sub.d*/ + "parse_r_%= __flags, %[flags] \n\t" + ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ + : [dst]"+r"(a0), + [flags]"=r"(eflags) + : [src]"r"(a1) + : "cc" + ); + break; + default: break; - case WRAP(JB): - case WRAP(JAE): - case WRAP(JE): - case WRAP(JNE): - case WRAP(JBE): - case WRAP(JA): - case WRAP(JL): - case WRAP(JGE): - case WRAP(JLE): - case WRAP(JG): - case WRAP(JS): - case WRAP(JNS): - case WRAP(JNO): - case WRAP(JO): + } + env->eflags = env->eflags & ~(0b100011010101); + env->eflags = env->eflags | (eflags & 0b100011010101); + } + break; + case INSTPTN_FLAGS_TEST: + { + bool is_same_reg = recovery & INSTPTN_FLAGS_SAME_REG; + uint64_t a0 = UC_GR(uc)[4]; + uint64_t a1 = is_same_reg ? a0 : UC_GR(uc)[5]; + uint64_t eflags = 0; + + switch (opnd_size) + { + case 8: + asm volatile ( + DEFINE_PARSE_R + "parse_r_%= __src, %[src] \n\t" + "parse_r_%= __dst, %[dst] \n\t" + ".word (0x003f8010 | (__src << 10) | (__dst << 5))\n\t" /*x86and.b*/ + "parse_r_%= __flags, %[flags] \n\t" + ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ + : [dst]"+r"(a0), + [flags]"=r"(eflags) + : [src]"r"(a1) + : "cc" + ); + break; + case 16: + asm volatile ( + DEFINE_PARSE_R + "parse_r_%= __src, %[src] \n\t" + "parse_r_%= __dst, %[dst] \n\t" + ".word (0x003f8011 | (__src << 10) | (__dst << 5))\n\t" /*x86and.h*/ + "parse_r_%= __flags, %[flags] \n\t" + ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ + : [dst]"+r"(a0), + [flags]"=r"(eflags) + : [src]"r"(a1) + : "cc" + ); + break; + case 32: + asm volatile ( + DEFINE_PARSE_R + "parse_r_%= __src, %[src] \n\t" + "parse_r_%= __dst, %[dst] \n\t" + ".word (0x003f8012 | (__src << 10) | (__dst << 5))\n\t" /*x86and.w*/ + "parse_r_%= __flags, %[flags] \n\t" + ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ + : [dst]"+r"(a0), + [flags]"=r"(eflags) + : [src]"r"(a1) + : "cc" + ); + break; + case 64: + asm volatile ( + DEFINE_PARSE_R + "parse_r_%= __src, %[src] \n\t" + "parse_r_%= __dst, %[dst] \n\t" + ".word (0x003f8013 | (__src << 10) | (__dst << 5))\n\t" /*x86and.d*/ + "parse_r_%= __flags, %[flags] \n\t" + ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ + : [dst]"+r"(a0), + [flags]"=r"(eflags) + : [src]"r"(a1) + : "cc" + ); + break; + default: break; - default: - lsassert(0); - break; - } - return; } + env->eflags = env->eflags & ~(0b100011000101); + env->eflags = env->eflags | (eflags & 0b100011000101); + } + break; + case INSTPTN_FLAGS_BT: + { + uint64_t a0 = UC_GR(uc)[4]; + uint64_t a1 = UC_GR(uc)[5]; + env->eflags = env->eflags & ~(1); // clear CF + env->eflags = env->eflags | ((a0 >> a1) & 1); // set CF + } + break; + case INSTPTN_FLAGS_NONE: + break; + default: + lsassert(0); + break; } } #undef WRAP diff --git a/target/i386/latx/translator/translate.c b/target/i386/latx/translator/translate.c index 7d6b364694..039ebe45f2 100644 --- a/target/i386/latx/translator/translate.c +++ b/target/i386/latx/translator/translate.c @@ -20,6 +20,7 @@ #include "latx-smc.h" #include "jrra.h" #include "latx-native-asm.h" +#include "latx-unwind.h" extern void *helper_tb_lookup_ptr(CPUArchState *); static int ss_generate_match_fail_native_code(void* code_buf); @@ -2497,6 +2498,11 @@ int tr_ir2_generate(struct TranslationBlock *tb) bool reduce_proepo = false; bool x87_fcsr_active = false; int softfpu_region_end = -1; +#ifdef CONFIG_LATX_INSTS_PATTERN + unsigned flags_recovery = INSTPTN_FLAGS_NONE; +#endif + + QEMU_BUILD_BUG_ON(CC_OP_NB > LATX_UNWIND_CC_OP_MASK + 1); lsenv->tr_data->softfpu_region_active = false; @@ -2586,11 +2592,14 @@ int tr_ir2_generate(struct TranslationBlock *tb) * at the end of TB translate cache. */ tcg_ctx->gen_insn_data[i][0] = pir1->info->address; - /* Preserve logical upper zeros if this instruction faults before - * the deferred clears at the normal TB exit have executed. The - * low 16 bits remain the condition-code operation. */ + /* Persist recovery state with the code, including loaded AOT TBs. */ tcg_ctx->gen_insn_data[i][1] = - (target_ulong)lsenv->tr_data->ymmh_zero_pending << 16; + (target_ulong)lsenv->tr_data->ymmh_zero_pending << + LATX_UNWIND_YMMH_SHIFT; +#ifdef CONFIG_LATX_INSTS_PATTERN + tcg_ctx->gen_insn_data[i][1] |= + flags_recovery << LATX_UNWIND_INSTPTN_SHIFT; +#endif #ifdef CONFIG_LATX_IMM_REG imm_cache->curr_ir1_index = i; @@ -2630,6 +2639,16 @@ int tr_ir2_generate(struct TranslationBlock *tb) } bool translation_success = ir1_translate(pir1); +#ifdef CONFIG_LATX_INSTS_PATTERN + if (translation_success) { + unsigned recovery = instptn_flags_recovery(pir1); + + /* A fault in the comparison itself still needs the old flags. */ + if (recovery != INSTPTN_FLAGS_NONE) { + flags_recovery = recovery; + } + } +#endif if (!translation_success) { #ifdef CONFIG_LATX_TU tb->s_data->tu_tb_mode = BAD_TB; diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index cfb1376b29..1fb3cb2dfb 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -37,6 +37,10 @@ #ifdef CONFIG_LATX #include "reg-map.h" #include "latx-options.h" +#include "latx-unwind.h" +#ifdef CONFIG_LATX_INSTS_PATTERN +#include "insts-pattern.h" +#endif #endif #define PREFIX_REPZ 0x01 @@ -8752,8 +8756,8 @@ static void restore_extcontext(CPUX86State *env, ucontext_t *uc) void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb, target_ulong *data) { - int cc_op = data[1] & 0xffff; - uint16_t ymmh_zero_pending = data[1] >> 16; + int cc_op = data[1] & LATX_UNWIND_CC_OP_MASK; + uint16_t ymmh_zero_pending = data[1] >> LATX_UNWIND_YMMH_SHIFT; /* env->eip = data[0] - tb->cs_base; */ env->eip = data[0]; @@ -8788,6 +8792,10 @@ void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb, env->regs[R_R15] = UC_GR(uc)[reg_gpr_map[r15_index]]; #endif restore_extcontext(env, uc); +#ifdef CONFIG_LATX_INSTS_PATTERN + opt_instptn_fix(env, (data[1] >> LATX_UNWIND_INSTPTN_SHIFT) & + LATX_UNWIND_INSTPTN_MASK); +#endif /* The saved host registers can still contain stale upper halves for * completed VEX.128 instructions whose clears were deferred. */ if (option_enable_lasx) { diff --git a/tests/integration/instptn-signal-recovery-i386.S b/tests/integration/instptn-signal-recovery-i386.S new file mode 100644 index 0000000000..e81c64eb54 --- /dev/null +++ b/tests/integration/instptn-signal-recovery-i386.S @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +.macro compare kind, bits, same=0 +.if \bits == 8 +.if \same + \kind %al, %al +.else + \kind %bl, %al +.endif +.elseif \bits == 16 +.if \same + \kind %ax, %ax +.else + \kind %bx, %ax +.endif +.else +.if \same + \kind %eax, %eax +.else + \kind %ebx, %eax +.endif +.endif +.endm + +.macro run_case name, kind, bits, mask, same=0 + mov $0x80008080, %eax + mov $0x7fff7f81, %ebx +.ifc \kind,bt + compare bt, \bits +.else + compare \kind, \bits, \same +.endif + pushfl + popl expected_flags + movl $\mask, flag_mask + movl $\name\()_fault, expected_pc + call arm_fault + call \name + call check_result +.endm + +.macro fault_function name, kind, bits, same=0 +.p2align 4 +\name: +.ifc \kind,bt + compare bt, \bits +.else + compare \kind, \bits, \same +.endif +\name\()_fault: + mov (%edi), %ecx +.ifc \kind,bt + jc 1f +.else + jz 1f +.endif + ret +1: + ret +.endm + +.text +.globl _start +_start: + mov $174, %eax + mov $11, %ebx + mov $action, %ecx + xor %edx, %edx + mov $8, %esi + int $0x80 + test %eax, %eax + js fail_setup + mov $192, %eax + xor %ebx, %ebx + mov $16384, %ecx + mov $3, %edx + mov $0x21, %esi + mov $-1, %edi + xor %ebp, %ebp + int $0x80 + cmp $-4095, %eax + jae fail_setup + mov $192, %eax + xor %ebx, %ebx + mov $65536, %ecx + mov $3, %edx + mov $0x22, %esi + mov $-1, %edi + xor %ebp, %ebp + int $0x80 + cmp $-4095, %eax + jae fail_setup + add $16383, %eax + and $-16384, %eax + mov %eax, page + movl $0x12345678, (%eax) + + movl $0xad7, expected_flags + movl $0x8d5, flag_mask + movl $plain_fault, expected_pc + call arm_fault + call plain + call check_result + run_case cmp8, cmp, 8, 0x8d5 + run_case cmp16, cmp, 16, 0x8d5 + run_case cmp32, cmp, 32, 0x8d5 + run_case test8, test, 8, 0x8c5 + run_case test16, test, 16, 0x8c5 + run_case test32, test, 32, 0x8c5 + run_case same8, test, 8, 0x8c5, 1 + run_case same16, test, 16, 0x8c5, 1 + run_case same32, test, 32, 0x8c5, 1 + run_case bt16, bt, 16, 1 + run_case bt32, bt, 32, 1 + xor %ebx, %ebx + jmp exit + +arm_fault: + movl $0, seen + mov $125, %eax + mov page, %ebx + mov $16384, %ecx + xor %edx, %edx + int $0x80 + test %eax, %eax + js fail_setup + pushl $0xad7 + popfl + mov $0x80008080, %eax + mov $0x7fff7f81, %ebx + mov $0x55667788, %ecx + mov page, %edi + ret + +check_result: + pushfl + pop %edx + xor expected_flags, %edx + and flag_mask, %edx + jnz fail_flags + cmpl $1, seen + jne fail_count + cmp $0x12345678, %ecx + jne fail_state + ret + +.p2align 4 +plain: + nop +plain_fault: + mov (%edi), %ecx + ret + fault_function cmp8, cmp, 8 + fault_function cmp16, cmp, 16 + fault_function cmp32, cmp, 32 + fault_function test8, test, 8 + fault_function test16, test, 16 + fault_function test32, test, 32 + fault_function same8, test, 8, 1 + fault_function same16, test, 16, 1 + fault_function same32, test, 32, 1 + fault_function bt16, bt, 16 + fault_function bt32, bt, 32 + +handler: + cmpl $11, 4(%esp) + jne fail_signal + cmpl $0, seen + jne fail_count + mov 8(%esp), %esi + mov 12(%esi), %eax + cmp page, %eax + jne fail_address + mov 12(%esp), %esi + mov 76(%esi), %eax + cmp expected_pc, %eax + jne fail_pc + mov 84(%esi), %eax + xor expected_flags, %eax + and flag_mask, %eax + jnz fail_flags + cmpl $0x80008080, 64(%esi) + jne fail_state + cmpl $0x7fff7f81, 52(%esi) + jne fail_state + cmpl $0x55667788, 60(%esi) + jne fail_state + mov 36(%esi), %eax + cmp page, %eax + jne fail_state + movl $1, seen + mov $125, %eax + mov page, %ebx + mov $16384, %ecx + mov $3, %edx + int $0x80 + test %eax, %eax + js fail_setup + ret + +restorer: + mov $173, %eax + int $0x80 + ud2 + +fail_setup: mov $1, %ebx; jmp exit +fail_signal: mov $2, %ebx; jmp exit +fail_address: mov $3, %ebx; jmp exit +fail_pc: mov $4, %ebx; jmp exit +fail_flags: mov $5, %ebx; jmp exit +fail_state: mov $6, %ebx; jmp exit +fail_count: mov $7, %ebx +exit: + mov $1, %eax + int $0x80 + +.data +.p2align 14 +action: .long handler, 0x04000004, restorer, 0, 0 +page: .long 0 +expected_pc: .long 0 +expected_flags: .long 0 +flag_mask: .long 0 +seen: .long 0 +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/instptn-signal-recovery.S b/tests/integration/instptn-signal-recovery.S new file mode 100644 index 0000000000..346df989d6 --- /dev/null +++ b/tests/integration/instptn-signal-recovery.S @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +.macro compare kind, bits, same=0 +.if \bits == 8 +.if \same + \kind %al, %al +.else + \kind %bl, %al +.endif +.elseif \bits == 16 +.if \same + \kind %ax, %ax +.else + \kind %bx, %ax +.endif +.elseif \bits == 32 +.if \same + \kind %eax, %eax +.else + \kind %ebx, %eax +.endif +.else +.if \same + \kind %rax, %rax +.else + \kind %rbx, %rax +.endif +.endif +.endm + +.macro run_case name, kind, bits, mask, same=0 + movabs $0x8000000080008080, %rax + movabs $0x7fffffff7fff7f81, %rbx +.ifc \kind,bt + compare bt, \bits +.else + compare \kind, \bits, \same +.endif + pushfq + popq expected_flags(%rip) + movq $\mask, flag_mask(%rip) + lea \name\()_fault(%rip), %rax + mov %rax, expected_pc(%rip) + call arm_fault + call \name + call check_result +.endm + +.macro fault_function name, kind, bits, same=0 +.p2align 4 +\name: +.ifc \kind,bt + compare bt, \bits +.else + compare \kind, \bits, \same +.endif +\name\()_fault: + mov (%rdi), %rcx +.ifc \kind,bt + jc 1f +.else + jz 1f +.endif + ret +1: + ret +.endm + +.text +.globl _start +_start: + mov $13, %eax + mov $11, %edi + lea action(%rip), %rsi + xor %edx, %edx + mov $8, %r10d + syscall + test %rax, %rax + js fail_setup + /* Shared mappings select the real parallel recovery path even without + * another thread. It must not depend on a fresh push/pop retranslation. */ + mov $9, %eax + xor %edi, %edi + mov $16384, %esi + mov $3, %edx + mov $0x21, %r10d + mov $-1, %r8 + xor %r9d, %r9d + syscall + test %rax, %rax + js fail_setup + mov $9, %eax + xor %edi, %edi + mov $65536, %esi + mov $3, %edx + mov $0x22, %r10d + mov $-1, %r8 + xor %r9d, %r9d + syscall + test %rax, %rax + js fail_setup + add $16383, %rax + and $-16384, %rax + mov %rax, page(%rip) + movq $0x12345678, (%rax) + movdqu poison(%rip), %xmm0 + + /* No delayed comparison: hot AOT must not consult a temporary IR1 list. */ + movq $0xad7, expected_flags(%rip) + movq $0x8d5, flag_mask(%rip) + lea plain_fault(%rip), %rax + mov %rax, expected_pc(%rip) + call arm_fault + call plain + call check_result + + run_case cmp8, cmp, 8, 0x8d5 + run_case cmp16, cmp, 16, 0x8d5 + run_case cmp32, cmp, 32, 0x8d5 + run_case cmp64, cmp, 64, 0x8d5 + run_case test8, test, 8, 0x8c5 + run_case test16, test, 16, 0x8c5 + run_case test32, test, 32, 0x8c5 + run_case test64, test, 64, 0x8c5 + run_case same8, test, 8, 0x8c5, 1 + run_case same16, test, 16, 0x8c5, 1 + run_case same32, test, 32, 0x8c5, 1 + run_case same64, test, 64, 0x8c5, 1 + run_case bt16, bt, 16, 1 + run_case bt32, bt, 32, 1 + run_case bt64, bt, 64, 1 + + xor %edi, %edi + jmp exit + +/* Protect a whole host page; no shared-page interpretation is involved. */ +arm_fault: + movl $0, seen(%rip) + mov $10, %eax + mov page(%rip), %rdi + mov $16384, %esi + xor %edx, %edx + syscall + test %rax, %rax + js fail_setup + pushq $0xad7 + popfq + movabs $0x8000000080008080, %rax + movabs $0x7fffffff7fff7f81, %rbx + movabs $0x1122334455667788, %rcx + ret + +check_result: + pushfq + pop %r8 + xor expected_flags(%rip), %r8 + and flag_mask(%rip), %r8 + jnz fail_flags + cmpl $1, seen(%rip) + jne fail_count + cmp $0x12345678, %rcx + jne fail_state + movdqu %xmm0, actual(%rip) + mov actual(%rip), %r8 + cmp poison(%rip), %r8 + jne fail_state + mov actual+8(%rip), %r8 + cmp poison+8(%rip), %r8 + jne fail_state + ret + +.p2align 4 +plain: + nop +plain_fault: + mov (%rdi), %rcx + ret + + fault_function cmp8, cmp, 8 + fault_function cmp16, cmp, 16 + fault_function cmp32, cmp, 32 + fault_function cmp64, cmp, 64 + fault_function test8, test, 8 + fault_function test16, test, 16 + fault_function test32, test, 32 + fault_function test64, test, 64 + fault_function same8, test, 8, 1 + fault_function same16, test, 16, 1 + fault_function same32, test, 32, 1 + fault_function same64, test, 64, 1 + fault_function bt16, bt, 16 + fault_function bt32, bt, 32 + fault_function bt64, bt, 64 + +handler: + cmp $11, %edi + jne fail_signal + cmpl $0, seen(%rip) + jne fail_count + mov 16(%rsi), %r8 + cmp page(%rip), %r8 + jne fail_address + mov 168(%rdx), %r8 + cmp expected_pc(%rip), %r8 + jne fail_pc + mov 176(%rdx), %r8 + xor expected_flags(%rip), %r8 + and flag_mask(%rip), %r8 + jnz fail_flags + movabs $0x8000000080008080, %r8 + cmp 144(%rdx), %r8 + jne fail_state + movabs $0x7fffffff7fff7f81, %r8 + cmp 128(%rdx), %r8 + jne fail_state + movabs $0x1122334455667788, %r8 + cmp 152(%rdx), %r8 + jne fail_state + mov 104(%rdx), %r8 + cmp page(%rip), %r8 + jne fail_state + mov 224(%rdx), %r9 + mov 160(%r9), %r8 + cmp poison(%rip), %r8 + jne fail_state + mov 168(%r9), %r8 + cmp poison+8(%rip), %r8 + jne fail_state + movl $1, seen(%rip) + mov $10, %eax + mov page(%rip), %rdi + mov $16384, %esi + mov $3, %edx + syscall + test %rax, %rax + js fail_setup + ret + +restorer: + mov $15, %eax + syscall + ud2 + +fail_setup: mov $1, %edi; jmp exit +fail_signal: mov $2, %edi; jmp exit +fail_address: mov $3, %edi; jmp exit +fail_pc: mov $4, %edi; jmp exit +fail_flags: mov $5, %edi; jmp exit +fail_state: mov $6, %edi; jmp exit +fail_count: mov $7, %edi +exit: + mov $60, %eax + syscall + +.data +/* Keep mutable test state out of the code's 16 KiB host page. */ +.p2align 14 +action: .quad handler, 0x04000004, restorer, 0 +poison: .quad 0x0123456789abcdef, 0xfedcba9876543210 +page: .quad 0 +expected_pc: .quad 0 +expected_flags: .quad 0 +flag_mask: .quad 0 +seen: .long 0 +.bss +.p2align 4 +actual: .skip 16 +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index 0e11dc576e..ea2c6a0d54 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -28,6 +28,14 @@ if 'x86_64-linux-user' in target_dirs 'timeout': 120, }] endif + latx_integration_tests += [{ + 'name': 'test-instptn-signal-recovery', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], + files('../../instptn-signal-recovery.S')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 120, + }] latx_integration_tests += [{ 'name': 'test-ymmh-conditional-save', 'runner': find_program('../../test-avx-regressions.py'), @@ -170,6 +178,14 @@ if 'i386-linux-user' in target_dirs 'timeout': 120, }] endif + latx_integration_tests += [{ + 'name': 'test-instptn-signal-recovery-i386', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': ['--guest-bits', '32', emulators['latx-i386'], + files('../../instptn-signal-recovery-i386.S')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 120, + }] latx_integration_tests += [{ 'name': 'test-ymmh-conditional-save-i386', 'runner': find_program('../../test-avx-regressions.py'), From 6d22cb62ac0e5f50ef1024355d90a3ee7ace867e Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 2 Nov 2026 12:22:39 +0800 Subject: [PATCH 16/33] LATX, opt: Remove redundant YMM clears after VMOVSS memory loads XV.PICKVE.W already zeroes bits 255:32 of the VMOVSS destination. Cancel its pending upper-half clear instead of scheduling another clear. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx-mov.c | 2 +- .../integration/registrations/avx/meson.build | 1 + tests/integration/vmovss-memory-zero.asm | 117 ++++++++++++++++++ 3 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 tests/integration/vmovss-memory-zero.asm diff --git a/target/i386/latx/translator/tr-avx-mov.c b/target/i386/latx/translator/tr-avx-mov.c index 886ca92633..925eed317d 100644 --- a/target/i386/latx/translator/tr-avx-mov.c +++ b/target/i386/latx/translator/tr-avx-mov.c @@ -320,7 +320,7 @@ bool translate_vmovss(IR1_INST *pir1) } else if (ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_mem(opnd1)) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); la_xvpickve_w(dest, src1, 0); - set_high128_xreg_to_zero(dest); + cancel_pending_ymmh_zero(dest); } else if (ir1_opnd_is_mem(opnd0) && ir1_opnd_is_xmm(opnd1)) { int little_disp; IR2_OPND mem_opnd = convert_mem(opnd0, &little_disp); diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index ea2c6a0d54..ca07e9f9f0 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -115,6 +115,7 @@ if 'x86_64-linux-user' in target_dirs files('../../avx-scalar-double-alias.asm'), files('../../vaddsubpd-lanes.asm'), files('../../vblendpd-xmm-alias.asm'), + files('../../vmovss-memory-zero.asm'), files('../../avx128-shuffle.S'), files('../../avx-ymm-writeback.S'), files('../../vpshufd-distinct-regs.S'), diff --git a/tests/integration/vmovss-memory-zero.asm b/tests/integration/vmovss-memory-zero.asm new file mode 100644 index 0000000000..3d8e781712 --- /dev/null +++ b/tests/integration/vmovss-memory-zero.asm @@ -0,0 +1,117 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro VERIFY 1 + vmovdqu [output], ymm%1 + mov eax, [output] + cmp eax, [input] + jne fail + cmp dword [output + 4], 0 + jne fail + mov rax, [output + 8] + or rax, [output + 16] + or rax, [output + 24] + jne fail + stmxcsr [status] + mov eax, [status] + cmp eax, [control] + jne fail +%endmacro + +%macro TEST 1 + vmovdqu ymm%1, [initial] + vmovss xmm%1, [input] + VERIFY %1 + ; Overwrite an older deferred upper-half clear before a TB boundary. + vmovdqu ymm%1, [initial] + vmovss xmm%1, xmm%1, xmm%1 + vmovss xmm%1, [input] + call %%next +%%next: + add rsp, 8 + VERIFY %1 +%endmacro + +section .text +_start: + xor r9d, r9d +.rounding: + mov eax, r9d + shl eax, 13 + or eax, 0x1fbf + mov [control], eax + ldmxcsr [control] + lea r10, [values] + mov r11d, 16 +.value: + mov eax, [r10] + mov [input], eax + %assign reg 0 + %rep 16 + TEST reg + %assign reg reg + 1 + %endrep + add r10, 4 + dec r11d + jnz .value + inc r9d + cmp r9d, 4 + jne .rounding + ; A four-byte load at a page end must not read the protected next page. + mov eax, 9 + xor edi, edi + mov esi, 8192 + mov edx, 3 + mov r10d, 0x22 + mov r8, -1 + xor r9d, r9d + syscall + cmp rax, -4095 + jae fail + mov r12, rax + mov dword [r12 + 4092], 0x7f800001 + lea rdi, [r12 + 4096] + mov esi, 4096 + xor edx, edx + mov eax, 10 + syscall + test eax, eax + jne fail + vmovdqu ymm0, [initial] + vmovss xmm0, [r12 + 4092] + mov dword [input], 0x7f800001 + VERIFY 0 + mov eax, 11 + mov rdi, r12 + mov esi, 8192 + syscall + test eax, eax + jne fail + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +initial: dq -1, 0x123456789abcdef0, -1, 0xfedcba9876543210 +values: + dd 0, 0x80000000, 1, 0x80000001 + dd 0x007fffff, 0x807fffff, 0x00800000, 0x80800000 + dd 0x3f800000, 0xbf800000, 0x7f7fffff, 0xff7fffff + dd 0x7f800000, 0xff800000, 0x7fc00001, 0x7f800001 + +section .bss align=32 +output: resb 32 +control: resd 1 +status: resd 1 +padding: resb 1 +input: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits From d9c2594955daf7f33cf8102778dc659d28da7d4d Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 2 Nov 2026 15:15:50 +0800 Subject: [PATCH 17/33] LATX, opt: Remove redundant YMM clears after VMOVD XMM writes The LASX sequences for VMOVD XMM destinations already write the full physical register and zero its upper bits. Cancel the pending upper-half clear instead of emitting an extra XV.PERMI.Q. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx-mov.c | 4 +- .../integration/registrations/avx/meson.build | 1 + tests/integration/vmovd-xmm-zero.asm | 98 +++++++++++++++++++ 3 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 tests/integration/vmovd-xmm-zero.asm diff --git a/target/i386/latx/translator/tr-avx-mov.c b/target/i386/latx/translator/tr-avx-mov.c index 925eed317d..3a1a36bbc8 100644 --- a/target/i386/latx/translator/tr-avx-mov.c +++ b/target/i386/latx/translator/tr-avx-mov.c @@ -348,7 +348,7 @@ bool translate_vmovd(IR1_INST *pir1) IR2_OPND src = load_freg_from_ir1_1(opnd1, false, IS_INTEGER); IR2_OPND dest = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd0)); la_xvpickve_w(dest, src, 0); - set_high128_xreg_to_zero(dest); + cancel_pending_ymmh_zero(dest); } else if (ir1_opnd_is_mem(opnd0) && ir1_opnd_is_xmm(opnd1)) { IR2_OPND src = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd1)); store_freg_to_ir1(src, opnd0, false, false); @@ -357,7 +357,7 @@ bool translate_vmovd(IR1_INST *pir1) IR2_OPND src = load_ireg_from_ir1(opnd1, UNKNOWN_EXTENSION, false); la_xvandi_b(dest, dest, 0); la_xvinsgr2vr_w(dest, src, 0); - set_high128_xreg_to_zero(dest); + cancel_pending_ymmh_zero(dest); } else if (ir1_opnd_is_gpr(opnd0) && ir1_opnd_is_xmm(opnd1)) { IR2_OPND dest = load_ireg_from_ir1(opnd0, UNKNOWN_EXTENSION, false); IR2_OPND src = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd1)); diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index ca07e9f9f0..d706b6e66a 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -115,6 +115,7 @@ if 'x86_64-linux-user' in target_dirs files('../../avx-scalar-double-alias.asm'), files('../../vaddsubpd-lanes.asm'), files('../../vblendpd-xmm-alias.asm'), + files('../../vmovd-xmm-zero.asm'), files('../../vmovss-memory-zero.asm'), files('../../avx128-shuffle.S'), files('../../avx-ymm-writeback.S'), diff --git a/tests/integration/vmovd-xmm-zero.asm b/tests/integration/vmovd-xmm-zero.asm new file mode 100644 index 0000000000..42163a06e0 --- /dev/null +++ b/tests/integration/vmovd-xmm-zero.asm @@ -0,0 +1,98 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro VERIFY 2 + vmovdqu [actual], ymm%1 + mov eax, [actual] + cmp eax, %2 + jne fail + mov rax, [actual + 4] + or rax, [actual + 12] + or rax, [actual + 20] + or eax, [actual + 28] + jne fail +%endmacro + +%macro GPR_TEST 1 + vmovdqu ymm%1, [junk] + vxorps xmm%1, xmm%1, xmm%1 + mov rax, -1 + mov eax, 0x89abcdef + vmovd xmm%1, eax + call %%next +%%next: + add rsp, 8 + VERIFY %1, 0x89abcdef +%endmacro + +%macro MEMORY_TEST 1 + vmovdqu ymm%1, [junk] + vmovss xmm%1, xmm%1, xmm%1 + mov dword [input + 1], 0xfedcba98 + vmovd xmm%1, [input + 1] + call %%next +%%next: + add rsp, 8 + VERIFY %1, 0xfedcba98 +%endmacro + +section .text +_start: + %assign reg 0 + %rep 16 + GPR_TEST reg + MEMORY_TEST reg + %assign reg reg + 1 + %endrep + + ; A four-byte load at a page end must not read the protected next page. + mov eax, 9 + xor edi, edi + mov esi, 8192 + mov edx, 3 + mov r10d, 0x22 + mov r8, -1 + xor r9d, r9d + syscall + cmp rax, -4095 + jae fail + mov r12, rax + mov dword [r12 + 4092], 0xfedcba98 + lea rdi, [r12 + 4096] + mov esi, 4096 + xor edx, edx + mov eax, 10 + syscall + test eax, eax + jne fail + vmovdqu ymm0, [junk] + vmovd xmm0, [r12 + 4092] + VERIFY 0, 0xfedcba98 + mov eax, 11 + mov rdi, r12 + mov esi, 8192 + syscall + test eax, eax + jne fail + + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +junk: times 4 dq 0xdeadbeefdeadbeef + +section .bss align=32 +actual: resb 32 +padding: resb 1 +input: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits From c7f005f484faf25932bc7e923087369d578769e2 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 2 Nov 2026 16:59:54 +0800 Subject: [PATCH 18/33] LATX, opt: Replicate same-source VPERM2F128 in one instruction When VPERM2F128 or VPERM2I128 has identical sources and selects the low 128-bit half for both result halves, use XVREPLVE0.Q instead of a copy followed by XVPERMI.Q. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx.c | 4 + .../integration/registrations/avx/meson.build | 1 + tests/integration/vperm2x128-immediate.asm | 98 +++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 tests/integration/vperm2x128-immediate.asm diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 0bf57f51dc..c1704d0991 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -2216,6 +2216,10 @@ bool translate_vperm2i128(IR1_INST * pir1) { IR2_OPND src2 = load_freg256_from_ir1(opnd2); IR2_OPND result = dest; + if (ir2_opnd_cmp(&src1, &src2) && (imm & 0xbb) == 0) { + la_xvreplve0_q(dest, src1); + return true; + } if (ir2_opnd_cmp(&dest, &src1) && !ir2_opnd_cmp(&src1, &src2)) result = ra_alloc_ftemp(); if (!ir2_opnd_cmp(&result, &src2)) diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index d706b6e66a..fe984a98f7 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -115,6 +115,7 @@ if 'x86_64-linux-user' in target_dirs files('../../avx-scalar-double-alias.asm'), files('../../vaddsubpd-lanes.asm'), files('../../vblendpd-xmm-alias.asm'), + files('../../vperm2x128-immediate.asm'), files('../../vmovd-xmm-zero.asm'), files('../../vmovss-memory-zero.asm'), files('../../avx128-shuffle.S'), diff --git a/tests/integration/vperm2x128-immediate.asm b/tests/integration/vperm2x128-immediate.asm new file mode 100644 index 0000000000..9f1442ba67 --- /dev/null +++ b/tests/integration/vperm2x128-immediate.asm @@ -0,0 +1,98 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +section .text + +%macro CASE 3 + vmovdqu ymm0, [junk] + vmovdqu ymm1, [input_a] + vmovdqu ymm2, [input_b] + %assign source_low 0 + %assign source_high 0 + %if %3 == 0 + %1 ymm0, ymm1, ymm2, %2 + %elif %3 == 1 + %1 ymm1, ymm1, ymm2, %2 + vmovdqu ymm0, ymm1 + %elif %3 == 2 + %1 ymm2, ymm1, ymm2, %2 + vmovdqu ymm0, ymm2 + %elif %3 == 3 + %assign source_low 2 + %assign source_high 2 + %1 ymm0, ymm1, ymm1, %2 + %elif %3 == 4 + %assign source_low 2 + %assign source_high 2 + %1 ymm1, ymm1, ymm1, %2 + vmovdqu ymm0, ymm1 + %elif %3 == 5 + %1 ymm0, ymm1, [input_b], %2 + %elif %3 == 6 + %1 ymm1, ymm1, [input_b], %2 + vmovdqu ymm0, ymm1 + %else + ; The low half is read after a VEX.128 write made upper zero pending. + %assign source_low 2 + %assign source_high 2 + vmovdqu xmm1, [input_a] + %1 ymm0, ymm1, ymm1, %2 + %endif + vmovdqu [actual], ymm0 + %assign half 0 + %rep 2 + %assign select ((%2 >> (half * 4)) & 3) + %assign clear ((%2 >> (half * 4 + 3)) & 1) + %if half == 0 + %assign same source_low + %else + %assign same source_high + %endif + %assign lane 0 + %rep 2 + %if clear || (%3 == 7 && (select & 1)) + xor eax, eax + %elif select < 2 || same + mov rax, [input_a + (select & 1) * 16 + lane * 8] + %else + mov rax, [input_b + (select & 1) * 16 + lane * 8] + %endif + cmp [actual + half * 16 + lane * 8], rax + jne fail + %assign lane lane + 1 + %endrep + %assign half half + 1 + %endrep +%endmacro + +_start: + mov edi, 1 + %assign immediate 0 + %rep 256 + %assign alias 0 + %rep 8 + CASE vperm2f128, immediate, alias + CASE vperm2i128, immediate, alias + %assign alias alias + 1 + %endrep + %assign immediate immediate + 1 + %endrep + xor edi, edi +fail: + mov eax, 60 + syscall + +section .rodata align=32 +input_a: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0x0123456789abcdef, 0x1020304050607080 +input_b: dq 0x7ff8123456789abc, 0xffffffffffffffff + dq 0xfedcba9876543210, 0x8877665544332211 +junk: times 4 dq 0xdeadbeefdeadbeef + +section .bss align=32 +actual: resb 32 + +section .note.GNU-stack noalloc noexec nowrite progbits From 6dd30e2b2f4c0806f9fd722e5d9adb96c7a81e29 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 2 Nov 2026 17:01:14 +0800 Subject: [PATCH 19/33] LATX, opt: Replicate YMM VPERMILPD lanes directly For immediate low nibbles 0x0 and 0xf, use XVREPL128VEI.D to duplicate the selected double in each 128-bit half. This replaces the generic shuffle and avoids its dependency on the previous destination value. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx.c | 4 + .../integration/registrations/avx/meson.build | 1 + tests/integration/vpermilpd-ymm-immediate.asm | 75 +++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 tests/integration/vpermilpd-ymm-immediate.asm diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index c1704d0991..d18c98241b 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -4649,6 +4649,10 @@ bool translate_vpermilpd(IR1_INST * pir1) { IR2_OPND src1 = load_freg256_from_ir1(opnd1); if (ir1_opnd_is_imm(opnd2)) { uint8_t imm = ir1_opnd_uimm(opnd2); + if ((imm & 0xf) == 0 || (imm & 0xf) == 0xf) { + la_xvrepl128vei_d(dest, src1, imm & 1); + return true; + } imm = 0b10100000 + ((imm & 0b00001000) << 3) + ((imm & 0b00000100) << 2) + ((imm & 0b00000010) << 1) + (imm & 0b00000001); la_xvpermi_d(dest, src1, imm); diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index fe984a98f7..fda955d2e8 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -116,6 +116,7 @@ if 'x86_64-linux-user' in target_dirs files('../../vaddsubpd-lanes.asm'), files('../../vblendpd-xmm-alias.asm'), files('../../vperm2x128-immediate.asm'), + files('../../vpermilpd-ymm-immediate.asm'), files('../../vmovd-xmm-zero.asm'), files('../../vmovss-memory-zero.asm'), files('../../avx128-shuffle.S'), diff --git a/tests/integration/vpermilpd-ymm-immediate.asm b/tests/integration/vpermilpd-ymm-immediate.asm new file mode 100644 index 0000000000..dcc42eafdb --- /dev/null +++ b/tests/integration/vpermilpd-ymm-immediate.asm @@ -0,0 +1,75 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro VERIFY 2 + vmovdqu [output], %1 + %assign lane 0 + %rep 4 + mov rax, [output + lane * 8] + %if %2 && lane >= 2 + test rax, rax + %else + cmp rax, [input + ((lane & 2) + ((imm >> lane) & 1)) * 8] + %endif + jne fail + %assign lane lane + 1 + %endrep +%endmacro + +section .text +_start: + mov ecx, 4 +.rounding: + mov eax, ecx + dec eax + shl eax, 13 + or eax, 0x1fbf + mov [mxcsr], eax + ldmxcsr [mxcsr] + %assign imm 0 + %rep 256 + vmovdqu ymm0, [junk] + vmovdqu ymm1, [input] + vpermilpd ymm0, ymm1, imm + VERIFY ymm0, 0 + vmovdqu ymm0, [input] + vpermilpd ymm0, ymm0, imm + VERIFY ymm0, 0 + vmovdqu ymm0, [junk] + vpermilpd ymm0, [input], imm + VERIFY ymm0, 0 + vmovdqu ymm1, [junk] + vmovdqu xmm1, [input] + vpermilpd ymm0, ymm1, imm + VERIFY ymm0, 1 + %assign imm imm + 1 + %endrep + stmxcsr [actual_mxcsr] + mov eax, [actual_mxcsr] + cmp eax, [mxcsr] + jne fail + dec ecx + jnz .rounding + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +input: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0x0123456789abcdef, 0xfedcba9876543210 +junk: times 4 dq 0xdeadbeefdeadbeef + +section .bss align=32 +output: resb 32 +mxcsr: resd 1 +actual_mxcsr: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits From ea6c0b00ebbf1e9a7f4a2783aa13b5854f59dacf Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 2 Nov 2026 17:01:47 +0800 Subject: [PATCH 20/33] LATX, opt: Replicate XMM VPERMILPD lanes directly When the low two immediate bits are 0x0 or 0x3, use VREPLVEI.D to duplicate the selected double. This replaces the generic shuffle and avoids its dependency on the previous destination value. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx.c | 8 +- .../integration/registrations/avx/meson.build | 1 + tests/integration/vpermilpd-immediate.asm | 73 +++++++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 tests/integration/vpermilpd-immediate.asm diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index d18c98241b..572c21f84b 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -4627,8 +4627,12 @@ bool translate_vpermilpd(IR1_INST * pir1) { IR2_OPND src1 = load_freg128_from_ir1(opnd1); if (ir1_opnd_is_imm(opnd2)) { uint8_t imm = ir1_opnd_uimm(opnd2); - imm = 0b10100000 + ((imm & 0b00001000) << 3) + ((imm & 0b00000100) << 2) + ((imm & 0b00000010) << 1) + (imm & 0b00000001); - la_xvpermi_d(dest, src1, imm); + if ((imm & 3) == 0 || (imm & 3) == 3) { + la_vreplvei_d(dest, src1, imm & 1); + } else { + imm = 0b10100000 + ((imm & 0b00001000) << 3) + ((imm & 0b00000100) << 2) + ((imm & 0b00000010) << 1) + (imm & 0b00000001); + la_xvpermi_d(dest, src1, imm); + } } else { IR2_OPND src2 = load_freg128_from_ir1(opnd2); diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index fda955d2e8..06f1115eec 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -117,6 +117,7 @@ if 'x86_64-linux-user' in target_dirs files('../../vblendpd-xmm-alias.asm'), files('../../vperm2x128-immediate.asm'), files('../../vpermilpd-ymm-immediate.asm'), + files('../../vpermilpd-immediate.asm'), files('../../vmovd-xmm-zero.asm'), files('../../vmovss-memory-zero.asm'), files('../../avx128-shuffle.S'), diff --git a/tests/integration/vpermilpd-immediate.asm b/tests/integration/vpermilpd-immediate.asm new file mode 100644 index 0000000000..bc7ddff26c --- /dev/null +++ b/tests/integration/vpermilpd-immediate.asm @@ -0,0 +1,73 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro VERIFY 1 + vmovdqu [output], %1 + mov rax, [output] + cmp rax, [input + (imm & 1) * 8] + jne fail + mov rax, [output + 8] + cmp rax, [input + ((imm >> 1) & 1) * 8] + jne fail + mov rax, [output + 16] + or rax, [output + 24] + jne fail +%endmacro + +section .text +_start: + mov ecx, 4 +.rounding: + mov eax, ecx + dec eax + shl eax, 13 + or eax, 0x1fbf + mov [mxcsr], eax + ldmxcsr [mxcsr] + %assign imm 0 + %rep 256 + vmovdqu ymm0, [junk] + vmovdqu ymm1, [input] + vpermilpd xmm0, xmm1, imm + VERIFY ymm0 + vmovdqu ymm0, [input] + vpermilpd xmm0, xmm0, imm + VERIFY ymm0 + vmovdqu ymm0, [junk] + vpermilpd xmm0, [input], imm + VERIFY ymm0 + ; A pending source upper-zero must not affect the low-lane shuffle. + vmovdqu xmm1, [input] + vpermilpd xmm0, xmm1, imm + VERIFY ymm0 + %assign imm imm + 1 + %endrep + stmxcsr [actual_mxcsr] + mov eax, [actual_mxcsr] + cmp eax, [mxcsr] + jne fail + dec ecx + jnz .rounding + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +input: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0x0123456789abcdef, 0xfedcba9876543210 +junk: times 4 dq 0xdeadbeefdeadbeef + +section .bss align=32 +output: resb 32 +mxcsr: resd 1 +actual_mxcsr: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits From 155c4058aa17d5f7e6733808628c381b5242c3a8 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Tue, 3 Nov 2026 17:33:29 +0800 Subject: [PATCH 21/33] LATX, opt: Replace the POPCNT bit loop with VPCNT.D Move the source into a temporary vector register, count its bits with VPCNT.D and move the low result back. This replaces the fixed-width loop that counts one bit per iteration. Keep the existing 16/32/64-bit destination writes and flag generation. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-mov.c | 46 +++++----------------------- 1 file changed, 8 insertions(+), 38 deletions(-) diff --git a/target/i386/latx/translator/tr-mov.c b/target/i386/latx/translator/tr-mov.c index b1c8abd097..e9918dc108 100644 --- a/target/i386/latx/translator/tr-mov.c +++ b/target/i386/latx/translator/tr-mov.c @@ -2142,52 +2142,22 @@ bool translate_popal(IR1_INST *pir1) { return true; } -/* -* int hacker_popcnt(uint32_t n) -* { -* n -= (n>>1) & 0x55555555; -* n = (n & 0x33333333) + ((n>>2) & 0x33333333); -* n = ((n>>4) + n) & 0x0F0F0F0F; -* n += n>>8; -* n += n>>16; -* return n & 0x0000003F; -* } -*/ bool translate_popcnt(IR1_INST *pir1) { IR2_OPND src_opnd = load_ireg_from_ir1(ir1_get_opnd(pir1, 1), ZERO_EXTENSION, false); IR2_OPND count = ra_alloc_itemp(); - IR2_OPND temp = ra_alloc_itemp(); - IR2_OPND src_temp = ra_alloc_itemp(); - IR2_OPND pop_count_ir2_opnd = ra_alloc_itemp(); - int pop_count = ir1_opnd_size(ir1_get_opnd(pir1, 0)); + IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND label_exit = ra_alloc_label(); - IR2_OPND label_zero = ra_alloc_label(); - IR2_OPND label_start = ra_alloc_label(); - - /* set O S Z A C P = 0 */ - la_x86mtflag(zero_ir2_opnd, 0x3f); - la_ori(count, zero_ir2_opnd, 0); - - la_beq(src_opnd, zero_ir2_opnd, label_zero); - - li_d(pop_count_ir2_opnd, pop_count); - - la_or(src_temp, zero_ir2_opnd, src_opnd); -/* label_start: */ - la_label(label_start); - la_andi(temp, src_temp, 1); - la_add_d(count, count, temp); - la_srli_d(src_temp, src_temp, 1); - la_addi_d(pop_count_ir2_opnd, pop_count_ir2_opnd, -1); - la_bne(pop_count_ir2_opnd, zero_ir2_opnd, label_start); + /* The zero-extended source occupies one independent 64-bit element. */ + la_movgr2fr_d(temp, src_opnd); + la_vpcnt_d(temp, temp); + la_movfr2gr_d(count, temp); ra_free_temp(temp); - /* 5. finish all steps, jump to exit */ - la_b(label_exit); -/* label_zero: */ - la_label(label_zero); + /* set O S Z A C P = 0 */ + la_x86mtflag(zero_ir2_opnd, 0x3f); + la_bne(src_opnd, zero_ir2_opnd, label_exit); /* set zf = 1 */ IR2_OPND n4095_opnd = ra_alloc_num_4095(); la_x86mtflag(n4095_opnd, 0x8); From a0002fa06de528b61ddcc3de7cb8b44f528aac25 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Tue, 3 Nov 2026 20:03:50 +0800 Subject: [PATCH 22/33] LATX, opt: Remove VINSERTF128 copies when dest is src1 When dest is src1, insert the selected 128-bit lane directly into dest instead of copying src1 to a temporary and copying the result back. Reuse the alias handling shared with VINSERTI128. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx.c | 20 +--- .../integration/registrations/avx/meson.build | 1 + tests/integration/vinsertf128-alias.asm | 94 +++++++++++++++++++ 3 files changed, 96 insertions(+), 19 deletions(-) create mode 100644 tests/integration/vinsertf128-alias.asm diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 572c21f84b..08796a19ed 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -1537,25 +1537,7 @@ bool translate_vinsertf128(IR1_INST * pir1) { if (!option_enable_lasx) { return translate_vinsertf128_lsx(pir1); } - - lsassert(ir1_opnd_num(pir1) == 4); - lsassert(ir1_opnd_is_ymm(ir1_get_opnd(pir1, 0)) && - ir1_opnd_is_ymm(ir1_get_opnd(pir1, 1)) && - ir1_opnd_size(ir1_get_opnd(pir1, 2)) == 128 && - ir1_opnd_is_imm(ir1_get_opnd(pir1, 3))); - IR2_OPND dest = load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); - IR2_OPND src1 = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - uint8 imm = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)) & 0x1; - IR2_OPND temp = ra_alloc_ftemp(); - la_xvori_b(temp, src1, 0); - if (!imm) { - la_xvpermi_q(temp, src2, VEXTRINS_IMM_4_0(3, 0)); - } else { - la_xvpermi_q(temp, src2, VEXTRINS_IMM_4_0(0, 2)); - } - la_xvori_b(dest, temp, 0); - return true; + return translate_vinserti128(pir1); } bool translate_vshufpd(IR1_INST * pir1) { diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index 06f1115eec..f1fc971972 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -115,6 +115,7 @@ if 'x86_64-linux-user' in target_dirs files('../../avx-scalar-double-alias.asm'), files('../../vaddsubpd-lanes.asm'), files('../../vblendpd-xmm-alias.asm'), + files('../../vinsertf128-alias.asm'), files('../../vperm2x128-immediate.asm'), files('../../vpermilpd-ymm-immediate.asm'), files('../../vpermilpd-immediate.asm'), diff --git a/tests/integration/vinsertf128-alias.asm b/tests/integration/vinsertf128-alias.asm new file mode 100644 index 0000000000..533f3b8475 --- /dev/null +++ b/tests/integration/vinsertf128-alias.asm @@ -0,0 +1,94 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +section .text + +%macro CHECK 2 + vmovdqu [actual], ymm0 + mov edi, %2 + mov rax, [%1] + cmp rax, [actual] + jne fail + mov rax, [%1 + 8] + cmp rax, [actual + 8] + jne fail + mov rax, [%1 + 16] + cmp rax, [actual + 16] + jne fail + mov rax, [%1 + 24] + cmp rax, [actual + 24] + jne fail +%endmacro + +%macro ALIASES 1 + lea rbx, [expected_ab_low] + lea rdx, [expected_aa_low] + test byte [immediate], 1 + jz %%expected + lea rbx, [expected_ab_high] + lea rdx, [expected_aa_high] +%%expected: + vmovdqu ymm0, [junk] + vmovdqu ymm1, [input_a] + vmovdqu ymm2, [input_b] + vinsertf128 ymm0, ymm1, xmm2, %1 + CHECK rbx, 1 + vmovdqu ymm0, [input_a] + vinsertf128 ymm0, ymm0, xmm2, %1 + CHECK rbx, 2 + vmovdqu ymm0, [input_b] + vinsertf128 ymm0, ymm1, xmm0, %1 + CHECK rbx, 3 + vinsertf128 ymm0, ymm1, xmm1, %1 + CHECK rdx, 4 + vmovdqu ymm0, [input_a] + vinsertf128 ymm0, ymm0, xmm0, %1 + CHECK rdx, 5 + vinsertf128 ymm0, ymm1, [input_b], %1 + CHECK rbx, 6 + vmovdqu ymm0, [input_a] + vinsertf128 ymm0, ymm0, [input_b], %1 + CHECK rbx, 7 +%endmacro + +_start: +%assign imm 0 +%rep 256 + mov byte [immediate], imm + ALIASES imm +%assign imm imm+1 +%endrep + ; A pending VEX upper-half clear must be applied before reading src1. + vmovdqu ymm0, [input_a] + vmovdqu ymm2, [input_b] + vpxor xmm0, xmm0, xmm0 + vinsertf128 ymm0, ymm0, xmm2, 0 + CHECK expected_pending, 8 + xor edi, edi +fail: + mov eax, 60 + syscall + +section .rodata align=32 +input_a: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0x123456789abcdef0, 0xfedcba9876543210 +input_b: dq 0xfff800000000cafe, 0x0123456789abcdef + dq 0xaaaaaaaa55555555, 0x9999999966666666 +junk: times 4 dq 0xdeadbeefdeadbeef +expected_ab_low: dq 0xfff800000000cafe, 0x0123456789abcdef + dq 0x123456789abcdef0, 0xfedcba9876543210 +expected_ab_high: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0xfff800000000cafe, 0x0123456789abcdef +expected_aa_low: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0x123456789abcdef0, 0xfedcba9876543210 +expected_aa_high: times 2 dq 0x7ff0000000000001, 0x8000000000000000 +expected_pending: dq 0xfff800000000cafe, 0x0123456789abcdef, 0, 0 + +section .bss align=32 +actual: resb 32 +immediate: resb 1 + +section .note.GNU-stack noalloc noexec nowrite progbits From 3ec116d86cc96d9bb118ed1cf89d146c5f3d57fb Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Tue, 3 Nov 2026 20:06:14 +0800 Subject: [PATCH 23/33] LATX, opt: Skip unused src1 upper-half clears in VINSERTF128 A high-lane insert reads only src1's low 128 bits and replaces its high half. Skip materializing src1's pending high-half clear and load it as XMM. Continue to materialize src2 before reading it. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx.c | 12 +- target/i386/latx/translator/translate.c | 5 + .../integration/registrations/avx/meson.build | 1 + .../integration/vinsertf128-high-pending.asm | 224 ++++++++++++++++++ 4 files changed, 239 insertions(+), 3 deletions(-) create mode 100644 tests/integration/vinsertf128-high-pending.asm diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 08796a19ed..578d641240 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -1507,10 +1507,16 @@ bool translate_vinserti128(IR1_INST * pir1) { ir1_opnd_is_ymm(ir1_get_opnd(pir1, 1)) && ir1_opnd_size(ir1_get_opnd(pir1, 2)) == 128 && ir1_opnd_is_imm(ir1_get_opnd(pir1, 3))); - IR2_OPND dest = load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); - IR2_OPND src1 = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); uint8 imm = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)) & 0x1; + bool high_insert_partial_read = imm && + ir1_opcode(pir1) == dt_X86_INS_VINSERTF128; + IR2_OPND dest = high_insert_partial_read ? + ra_alloc_xmm(ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0))) : + load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); + IR2_OPND src1 = high_insert_partial_read ? + ra_alloc_xmm(ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) : + load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); + IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); bool dest_is_src1 = ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) == ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1)); bool dest_is_src2 = ir1_opnd_is_xmm(ir1_get_opnd(pir1, 2)) && diff --git a/target/i386/latx/translator/translate.c b/target/i386/latx/translator/translate.c index 039ebe45f2..e665807cb1 100644 --- a/target/i386/latx/translator/translate.c +++ b/target/i386/latx/translator/translate.c @@ -1925,6 +1925,11 @@ bool ir1_translate(IR1_INST *ir1) IR1_OPND *opnd = ir1_get_opnd(ir1, i); if (ir1_opnd_is_ymm(opnd)) { + /* A high insert reads only src1's low 128 bits. */ + if (i == 1 && ir1_opcode(ir1) == dt_X86_INS_VINSERTF128 && + (ir1_opnd_uimm(ir1_get_opnd(ir1, 3)) & 1)) { + continue; + } flush_pending_ymmh_zero( ra_alloc_xmm(ir1_opnd_base_reg_num(opnd))); } else if (ir1_opnd_is_mem(opnd) && diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index f1fc971972..bcc95c8000 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -116,6 +116,7 @@ if 'x86_64-linux-user' in target_dirs files('../../vaddsubpd-lanes.asm'), files('../../vblendpd-xmm-alias.asm'), files('../../vinsertf128-alias.asm'), + files('../../vinsertf128-high-pending.asm'), files('../../vperm2x128-immediate.asm'), files('../../vpermilpd-ymm-immediate.asm'), files('../../vpermilpd-immediate.asm'), diff --git a/tests/integration/vinsertf128-high-pending.asm b/tests/integration/vinsertf128-high-pending.asm new file mode 100644 index 0000000000..9139c8dbab --- /dev/null +++ b/tests/integration/vinsertf128-high-pending.asm @@ -0,0 +1,224 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +%macro CHECK 2 + vmovdqu [actual], %1 + lea rsi, [actual] + lea rdi, [%2] + mov ecx, 4 +%%next: + mov rax, [rsi] + cmp rax, [rdi] + jne fail + add rsi, 8 + add rdi, 8 + loop %%next +%endmacro + +%macro INSERT 2 + vmovdqu ymm0, [input_b] + vmovdqu ymm1, [input_a] + vmovdqu ymm2, [input_b] + vmovdqa xmm1, xmm1 + vmovdqa xmm0, xmm0 + %if %2 = 0 + vinsertf128 ymm0, ymm1, xmm2, %1 + %elif %2 = 1 + vmovdqa xmm0, xmm1 + vinsertf128 ymm0, ymm0, xmm2, %1 + %elif %2 = 2 + vinsertf128 ymm0, ymm1, xmm0, %1 + %elif %2 = 3 + vinsertf128 ymm0, ymm1, xmm1, %1 + %elif %2 = 4 + vmovdqa xmm0, xmm1 + vinsertf128 ymm0, ymm0, xmm0, %1 + %elif %2 = 5 + vinsertf128 ymm0, ymm1, [input_b], %1 + %else + vmovdqa xmm0, xmm1 + vinsertf128 ymm0, ymm0, [input_b], %1 + %endif + %if %2 = 3 || %2 = 4 + %if %1 & 1 + CHECK ymm0, expected_aa_high + %else + CHECK ymm0, expected_a_zero + %endif + %else + %if %1 & 1 + CHECK ymm0, expected_ab_high + %else + CHECK ymm0, expected_b_zero + %endif + %endif + CHECK ymm1, expected_a_zero +%endmacro + +; %1 selects distinct, src1 alias or src2 alias. Read exactly 16 bytes. +%macro MEMORY 2 + vmovdqu ymm0, [input_b] + vmovdqu ymm1, [input_a] + lea rdi, [r13 - %2] + vmovdqa xmm0, xmm0 + vmovdqa xmm1, xmm1 + %if %1 = 1 + vinsertf128 ymm0, ymm0, [rdi], 255 + CHECK ymm0, expected_bb_high + %else + vinsertf128 ymm0, ymm1, [rdi], 255 + CHECK ymm0, expected_ab_high + %endif + CHECK ymm1, expected_a_zero +%endmacro + +section .text +_start: + stmxcsr [original_mxcsr] +%assign imm 0 +%rep 256 +%assign alias 0 +%rep 7 + INSERT imm, alias +%assign alias alias + 1 +%endrep +%assign imm imm + 1 +%endrep + stmxcsr [actual_mxcsr] + mov eax, [original_mxcsr] + cmp eax, [actual_mxcsr] + jne fail + + mov eax, 9 + xor edi, edi + mov esi, 131072 + mov edx, 3 + mov r10d, 0x22 + mov r8, -1 + xor r9d, r9d + syscall + cmp rax, -4095 + jae fail + mov r12, rax + lea r13, [rax + 65536] + mov rax, [input_b] + mov [r13 - 16], rax + mov rax, [input_b + 8] + mov [r13 - 8], rax + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + xor edx, edx + syscall + test eax, eax + jne fail + MEMORY 0, 16 + MEMORY 1, 16 + + mov eax, 13 + mov edi, 11 + lea rsi, [action] + xor edx, edx + mov r10d, 8 + syscall + test eax, eax + jne fail +%assign alias 0 +%rep 2 + ; Reprotect before each cross-page load, then recreate deferred clears. + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + mov edx, 3 + syscall + mov rax, [input_b] + mov [r13 - 15], rax + mov rax, [input_b + 8] + mov [r13 - 7], rax + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + xor edx, edx + syscall + test eax, eax + jne fail + MEMORY alias, 15 +%assign alias alias + 1 +%endrep + cmp dword [faults], 2 + jne fail + mov eax, 11 + mov rdi, r12 + mov esi, 131072 + syscall + xor edi, edi + jmp exit + +handler: + cmp edi, 11 + jne fail + cmp [rsi + 16], r13 + jne fail + ; x86-64 ucontext holds fpregs at 224; XSAVE YMM_Hi128 starts at 576. + mov r8, [rdx + 224] + mov rax, [input_b] + cmp [r8 + 160], rax + jne fail + mov rax, [input_b + 8] + cmp [r8 + 168], rax + jne fail + mov rax, [input_a] + cmp [r8 + 176], rax + jne fail + mov rax, [input_a + 8] + cmp [r8 + 184], rax + jne fail + cmp qword [r8 + 576], 0 + jne fail + cmp qword [r8 + 584], 0 + jne fail + cmp qword [r8 + 592], 0 + jne fail + cmp qword [r8 + 600], 0 + jne fail + inc dword [faults] + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + mov edx, 3 + syscall + test eax, eax + jne fail + ret +restorer: + mov eax, 15 + syscall +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +input_a: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0x123456789abcdef0, 0xfedcba9876543210 +input_b: dq 0xfff800000000cafe, 0x0123456789abcdef + dq 0xaaaaaaaa55555555, 0x9999999966666666 +expected_a_zero: dq 0x7ff0000000000001, 0x8000000000000000, 0, 0 +expected_b_zero: dq 0xfff800000000cafe, 0x0123456789abcdef, 0, 0 +expected_ab_high: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0xfff800000000cafe, 0x0123456789abcdef +expected_aa_high: times 2 dq 0x7ff0000000000001, 0x8000000000000000 +expected_bb_high: times 2 dq 0xfff800000000cafe, 0x0123456789abcdef +action: dq handler, 0x04000004, restorer, 0 + +section .bss align=32 +actual: resb 32 +original_mxcsr: resd 1 +actual_mxcsr: resd 1 +faults: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits From 49e6b17dce2824aa10940375308fbf4d0e7730f7 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Tue, 3 Nov 2026 20:09:08 +0800 Subject: [PATCH 24/33] LATX, fix: Avoid oversized VBROADCASTSD memory reads The memory operand is eight bytes, but translation loads 16 bytes before broadcasting the low double. A valid page-end operand can therefore fault. Use XVLDREPL.D when the displacement is a multiple of eight. Otherwise, load one double with FLD.D and replicate it. Keep the page check at eight bytes. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx.c | 23 ++- .../integration/registrations/avx/meson.build | 1 + tests/integration/vbroadcastsd-memory.asm | 163 ++++++++++++++++++ 3 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 tests/integration/vbroadcastsd-memory.asm diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 578d641240..185833cc73 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -1298,7 +1298,28 @@ bool translate_vbroadcastsd(IR1_INST * pir1) { lsassert(ir1_opnd_is_ymm(ir1_get_opnd(pir1, 0))); IR2_OPND dest = load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); - IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); + IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); + + if (ir1_opnd_is_mem(opnd1)) { + int disp; + IR2_OPND address = convert_mem(opnd1, &disp); + + gen_test_page_flag(address, disp, PAGE_READ, 8); + /* XVLDREPL.D encodes its signed displacement in units of 8 bytes. */ + if (!(disp & 7)) { + la_xvldrepl_d(dest, address, disp / 8); + } else { + IR2_OPND src = ra_alloc_ftemp(); + + la_fld_d(src, address, disp); + la_xvreplve0_d(dest, src); + ra_free_temp(src); + } + ra_free_mem_temp(address); + return true; + } + + IR2_OPND src = load_freg128_from_ir1(opnd1); la_xvreplve0_d(dest, src); return true; } diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index bcc95c8000..0fcdbc8a13 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -115,6 +115,7 @@ if 'x86_64-linux-user' in target_dirs files('../../avx-scalar-double-alias.asm'), files('../../vaddsubpd-lanes.asm'), files('../../vblendpd-xmm-alias.asm'), + files('../../vbroadcastsd-memory.asm'), files('../../vinsertf128-alias.asm'), files('../../vinsertf128-high-pending.asm'), files('../../vperm2x128-immediate.asm'), diff --git a/tests/integration/vbroadcastsd-memory.asm b/tests/integration/vbroadcastsd-memory.asm new file mode 100644 index 0000000000..ef7b94cbf8 --- /dev/null +++ b/tests/integration/vbroadcastsd-memory.asm @@ -0,0 +1,163 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +%macro CHECK 0 + vmovdqu [actual], ymm0 + mov rax, [rdi] + lea rsi, [actual] + mov ecx, 4 +%%loop: + cmp [rsi], rax + jne fail + add rsi, 8 + loop %%loop +%endmacro + +%macro LOAD_DISP 1 + lea rbx, [data + 64 - %1] + lea rdi, [rbx + %1] + vbroadcastsd ymm0, [rbx + %1] + CHECK +%endmacro + +section .text +_start: + stmxcsr [original_mxcsr] + vmovdqu ymm1, [sentinel] + LOAD_DISP -4096 + LOAD_DISP -2048 + LOAD_DISP -2040 + LOAD_DISP -9 + LOAD_DISP -8 + LOAD_DISP -1 + LOAD_DISP 0 + LOAD_DISP 1 + LOAD_DISP 8 + LOAD_DISP 2040 + LOAD_DISP 2047 + LOAD_DISP 4096 + +%assign offset 0 +%rep 8 + lea rbx, [data + offset] + lea rdi, [rbx + 8] + vxorps xmm0, xmm0, xmm0 + vbroadcastsd ymm0, [rbx + 8] + CHECK +%assign offset offset + 1 +%endrep + + ; A register source must still work when its upper half is pending zero. + vmovdqu ymm0, [sentinel] + vmovdqa xmm0, [data] + vbroadcastsd ymm0, xmm0 + lea rdi, [data] + CHECK + vbroadcastsd ymm0, xmm1 + lea rdi, [sentinel] + CHECK + vmovdqu [actual], ymm1 + lea rsi, [actual] + lea rdi, [sentinel] + mov ecx, 4 +.preserved: + mov rax, [rsi] + cmp rax, [rdi] + jne fail + add rsi, 8 + add rdi, 8 + loop .preserved + stmxcsr [actual_mxcsr] + mov eax, [original_mxcsr] + cmp eax, [actual_mxcsr] + jne fail + + ; Leave the last 8 bytes accessible, followed by a protected page. + mov eax, 9 + xor edi, edi + mov esi, 131072 + mov edx, 3 + mov r10d, 0x22 + mov r8, -1 + xor r9d, r9d + syscall + cmp rax, -4095 + jae fail + mov r12, rax + lea r13, [rax + 65536] + mov rax, 0x7ff0000000000001 + mov [r13 - 8], rax + mov rax, 0xfedcba9876543210 + mov [r13], rax + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + xor edx, edx + syscall + test eax, eax + jne fail + + lea rdi, [r13 - 8] + vbroadcastsd ymm0, [rdi] + CHECK + + ; A genuine cross-page fault must be restartable with the right address. + mov eax, 13 + mov edi, 11 + lea rsi, [action] + xor edx, edx + mov r10d, 8 + syscall + test eax, eax + jne fail + lea rdi, [r13 - 7] + vbroadcastsd ymm0, [rdi] + CHECK + cmp dword [faults], 1 + jne fail + mov eax, 11 + mov rdi, r12 + mov esi, 131072 + syscall + xor edi, edi + jmp exit + +handler: + cmp edi, 11 + jne fail + cmp [rsi + 16], r13 + jne fail + inc dword [faults] + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + mov edx, 3 + syscall + test eax, eax + jne fail + ret +restorer: + mov eax, 15 + syscall +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +sentinel: dq 0x0123456789abcdef, 0xfedcba9876543210 + dq 0x7ff0000000000001, 0x8000000000000000 +data: times 16 dq 0x7ff0000000000001, 0x8000000000000000 +action: dq handler, 0x04000004, restorer, 0 + +section .bss align=32 +actual: resb 32 +original_mxcsr: resd 1 +actual_mxcsr: resd 1 +faults: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits From 5741440e1303fab99158ce75e3d72cec8f6db5d0 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Tue, 3 Nov 2026 20:11:17 +0800 Subject: [PATCH 25/33] LATX, opt: Avoid VCVTSS2SD result copies when dest is src1 Insert the converted low double directly into dest when it is src1. The high double already holds the required source bits, so the temporary shuffle and full-register copy are unnecessary. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx-cvt.c | 8 +- .../integration/registrations/avx/meson.build | 1 + tests/integration/vcvtss2sd-alias.asm | 110 ++++++++++++++++++ 3 files changed, 116 insertions(+), 3 deletions(-) create mode 100644 tests/integration/vcvtss2sd-alias.asm diff --git a/target/i386/latx/translator/tr-avx-cvt.c b/target/i386/latx/translator/tr-avx-cvt.c index 96b15dcbe0..49fae2cd0f 100644 --- a/target/i386/latx/translator/tr-avx-cvt.c +++ b/target/i386/latx/translator/tr-avx-cvt.c @@ -637,9 +637,11 @@ bool translate_vcvtss2sd(IR1_INST * pir1) { ra_alloc_ftemp() : dest; la_fcvt_d_s(result, src2); - la_vshuf4i_d(result, src1, 0xc); - if (!ir2_opnd_cmp(&result, &dest)) - la_xvori_b(dest, result, 0); + if (ir2_opnd_cmp(&dest, &src1)) { + la_xvinsve0_d(dest, result, 0); + } else { + la_vshuf4i_d(dest, src1, 0xc); + } set_high128_xreg_to_zero(dest); return true; } diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index 0fcdbc8a13..8b6cf4dc34 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -129,6 +129,7 @@ if 'x86_64-linux-user' in target_dirs files('../../vroundps-imm0b.S'), files('../../vroundsd-fixed.asm'), files('../../vroundss-fixed.asm'), + files('../../vcvtss2sd-alias.asm'), files('../../vcvtdq2pd-ymm-alias.asm'), files('../../avx-regression-extract-alias.S'), files('../../avx-regression-narrow-state.S'), diff --git a/tests/integration/vcvtss2sd-alias.asm b/tests/integration/vcvtss2sd-alias.asm new file mode 100644 index 0000000000..5405112175 --- /dev/null +++ b/tests/integration/vcvtss2sd-alias.asm @@ -0,0 +1,110 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro VERIFY 1 + vmovdqu [output], %1 + mov rax, [output] + cmp rax, [r10 + 8] + jne fail + mov rax, [output + 8] + cmp rax, [initial + 8] + jne fail + mov rax, [output + 16] + or rax, [output + 24] + jne fail +%endmacro + +%macro TEST 0 + vmovdqu ymm0, [initial] + vmovdqu ymm1, [initial] + vmovdqu ymm2, [initial] + vcvtss2sd xmm0, xmm1, xmm2 + VERIFY ymm0 + vmovdqu ymm1, [initial] + vcvtss2sd xmm1, xmm1, xmm2 + VERIFY ymm1 + vmovdqu ymm2, [initial] + vcvtss2sd xmm2, xmm1, xmm2 + VERIFY ymm2 + vmovdqu ymm1, [initial] + vcvtss2sd xmm0, xmm1, xmm1 + VERIFY ymm0 + vcvtss2sd xmm1, xmm1, xmm1 + VERIFY ymm1 + vmovdqu ymm1, [initial] + vcvtss2sd xmm0, xmm1, [initial] + VERIFY ymm0 + vcvtss2sd xmm1, xmm1, [initial] + VERIFY ymm1 +%endmacro + +section .text +_start: + xor r9d, r9d +.rounding: + mov eax, r9d + shl eax, 13 + or eax, 0x1f80 + mov [mxcsr], eax + ldmxcsr [mxcsr] + lea r10, [cases] + mov r11d, 12 +.case: + mov eax, [r10] + mov [initial], eax + TEST + add r10, 16 + dec r11d + jnz .case + inc r9d + cmp r9d, 4 + jne .rounding + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=8 +cases: + dd 0, 0 + dq 0 + dd 0x80000000, 0 + dq 0x8000000000000000 + dd 0x3f000000, 0 + dq 0x3fe0000000000000 + dd 0xbf000000, 0 + dq 0xbfe0000000000000 + dd 0x3fc00000, 0 + dq 0x3ff8000000000000 + dd 0xbfc00000, 0 + dq 0xbff8000000000000 + dd 0x00800000, 0 + dq 0x3810000000000000 + dd 0x80800000, 0 + dq 0xb810000000000000 + dd 0x7f7fffff, 0 + dq 0x47efffffe0000000 + dd 0xff7fffff, 0 + dq 0xc7efffffe0000000 + dd 0x7f800000, 0 + dq 0x7ff0000000000000 + dd 0xff800000, 0 + dq 0xfff0000000000000 + +section .data align=32 +initial: + dd 0, 0x89abcdef + dq 0x123456789abcdef0, -1, -1 + +section .bss align=32 +output: resb 32 +mxcsr: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits From 0fdd36e97c164a54317caa2668ef03c879463ab6 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Tue, 3 Nov 2026 20:13:29 +0800 Subject: [PATCH 26/33] LATX, opt: Remove redundant VMOVSD register copies Handle three-register VMOVSD aliases directly. Identical sources need only a copy when dest differs; dest equal to src2 needs only src1's high double inserted. For other cases, merge the source lanes directly in dest instead of copying a temporary result back. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx-mov.c | 21 ++-- .../integration/registrations/avx/meson.build | 1 + tests/integration/vmovsd-alias.asm | 109 ++++++++++++++++++ 3 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 tests/integration/vmovsd-alias.asm diff --git a/target/i386/latx/translator/tr-avx-mov.c b/target/i386/latx/translator/tr-avx-mov.c index 3a1a36bbc8..815a1ee950 100644 --- a/target/i386/latx/translator/tr-avx-mov.c +++ b/target/i386/latx/translator/tr-avx-mov.c @@ -635,14 +635,19 @@ bool translate_vmovsd(IR1_INST * pir1) { IR2_OPND temp_src1 = load_freg128_from_ir1(src); IR2_OPND temp_src2 = load_freg128_from_ir1(src2); - IR2_OPND result = ir2_opnd_cmp(&temp_dest, &temp_src2) ? - ra_alloc_ftemp() : temp_dest; - - if (!ir2_opnd_cmp(&result, &temp_src1)) - la_vori_b(result, temp_src1, 0x0); - la_vshuf4i_d(result, temp_src2, 0b00000110); - if (!ir2_opnd_cmp(&result, &temp_dest)) - la_xvori_b(temp_dest, result, 0x0); + + if (ir2_opnd_cmp(&temp_src1, &temp_src2)) { + if (!ir2_opnd_cmp(&temp_dest, &temp_src1)) { + la_vori_b(temp_dest, temp_src1, 0); + } + } else if (ir2_opnd_cmp(&temp_dest, &temp_src2)) { + la_vextrins_d(temp_dest, temp_src1, VEXTRINS_IMM_4_0(1, 1)); + } else { + if (!ir2_opnd_cmp(&temp_dest, &temp_src1)) { + la_vori_b(temp_dest, temp_src1, 0); + } + la_vshuf4i_d(temp_dest, temp_src2, 0b00000110); + } set_high128_xreg_to_zero(temp_dest); return true; } diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index 8b6cf4dc34..efb74ead93 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -118,6 +118,7 @@ if 'x86_64-linux-user' in target_dirs files('../../vbroadcastsd-memory.asm'), files('../../vinsertf128-alias.asm'), files('../../vinsertf128-high-pending.asm'), + files('../../vmovsd-alias.asm'), files('../../vperm2x128-immediate.asm'), files('../../vpermilpd-ymm-immediate.asm'), files('../../vpermilpd-immediate.asm'), diff --git a/tests/integration/vmovsd-alias.asm b/tests/integration/vmovsd-alias.asm new file mode 100644 index 0000000000..008a97ab83 --- /dev/null +++ b/tests/integration/vmovsd-alias.asm @@ -0,0 +1,109 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +section .text + +%macro CASE 4 + vmovdqu ymm%1, [pattern_a] + vmovdqu ymm%2, [pattern_b] + vmovdqu ymm%3, [pattern_c] +%if %4 + vxorps xmm%2, xmm%2, xmm%2 +%endif + vmovdqu [saved1], ymm%2 + vmovdqu [saved2], ymm%3 + mov rax, [saved2] + mov [expected], rax + mov rax, [saved1 + 8] + mov [expected + 8], rax + mov qword [expected + 16], 0 + mov qword [expected + 24], 0 +%if %4 + vxorps xmm%2, xmm%2, xmm%2 +%endif + stmxcsr [mxcsr_before] + vmovsd xmm%1, xmm%2, xmm%3 + vmovdqu [actual], ymm%1 + stmxcsr [mxcsr_after] + inc edi + mov eax, [mxcsr_before] + cmp eax, [mxcsr_after] + jne fail +%assign lane 0 +%rep 4 + mov rax, [actual + lane * 8] + cmp rax, [expected + lane * 8] + jne fail +%assign lane lane + 1 +%endrep +%if %2 != %1 + vmovdqu [actual], ymm%2 +%assign lane 0 +%rep 4 + mov rax, [actual + lane * 8] + cmp rax, [saved1 + lane * 8] + jne fail +%assign lane lane + 1 +%endrep +%endif +%if %3 != %1 + vmovdqu [actual], ymm%3 +%assign lane 0 +%rep 4 + mov rax, [actual + lane * 8] + cmp rax, [saved2 + lane * 8] + jne fail +%assign lane lane + 1 +%endrep +%endif +%endmacro + +_start: + xor edi, edi +%assign rc 0 +%rep 4 + mov dword [mxcsr_before], 0x1f80 | (rc << 13) + ldmxcsr [mxcsr_before] +%assign d 0 +%rep 16 +%assign s1 (d + 1) % 16 +%assign s2 (d + 2) % 16 + CASE d, s1, s2, 0 + CASE d, d, s2, 0 + CASE d, s1, d, 0 + CASE d, s1, s1, 0 + CASE d, d, d, 0 + CASE d, s1, s2, 1 + CASE d, d, s2, 1 + CASE d, s1, d, 1 + CASE d, s1, s1, 1 + CASE d, d, d, 1 +%assign d d + 1 +%endrep +%assign rc rc + 1 +%endrep + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +pattern_a: dq 0x8000000000000000, 0x7ff0000000000001, -1, 0x123456789abcdef0 +pattern_b: dq 0x7ff8000000000055, 0x0123456789abcdef, 0x7777777777777777, -1 +pattern_c: dq 0x7ff0000000000001, 0x5555555555555555, -1, -1 + +section .bss align=32 +actual: resb 32 +expected: resb 32 +saved1: resb 32 +saved2: resb 32 +mxcsr_before: resd 1 +mxcsr_after: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits From 18e25368ed787093c22c4dff47b9469b676cca1a Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Tue, 3 Nov 2026 20:15:58 +0800 Subject: [PATCH 27/33] LATX, opt: Interleave YMM VSHUFPD lanes directly For effective immediates 0x0 and 0xf, use XVILVL.D or XVILVH.D to produce the result directly. For 0x3 and 0xc, combine the two interleaves with one XVPERMI.Q. This avoids the generic two-temporary shuffle sequence. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx.c | 22 ++++- .../integration/registrations/avx/meson.build | 1 + tests/integration/vshufpd-interleave.asm | 88 +++++++++++++++++++ 3 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 tests/integration/vshufpd-interleave.asm diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 185833cc73..5a39d821e5 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -1581,10 +1581,30 @@ bool translate_vshufpd(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); + uint8_t imm8 = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)) & 0xf; + + if ((imm8 & 0x5) == ((imm8 >> 1) & 0x5)) { + if (imm8 == 0) { + la_xvilvl_d(dest, src2, src1); + } else if (imm8 == 0xf) { + la_xvilvh_d(dest, src2, src1); + } else { + IR2_OPND low = ra_alloc_ftemp(); + + if (imm8 == 0xc) { + la_xvilvl_d(low, src2, src1); + la_xvilvh_d(dest, src2, src1); + } else { + la_xvilvh_d(low, src2, src1); + la_xvilvl_d(dest, src2, src1); + } + la_xvpermi_q(dest, low, VEXTRINS_IMM_4_0(3, 0)); + } + return true; + } IR2_OPND temp_l = ra_alloc_ftemp(); IR2_OPND result = ir2_opnd_cmp(&dest, &src2) ? ra_alloc_ftemp() : dest; - uint8_t imm8 = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)) & 0xf; uint8_t l = imm8 & 0x3; uint8_t h = imm8 >> 2; if (l == 0) { diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index efb74ead93..65428276a0 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -132,6 +132,7 @@ if 'x86_64-linux-user' in target_dirs files('../../vroundss-fixed.asm'), files('../../vcvtss2sd-alias.asm'), files('../../vcvtdq2pd-ymm-alias.asm'), + files('../../vshufpd-interleave.asm'), files('../../avx-regression-extract-alias.S'), files('../../avx-regression-narrow-state.S'), files('../../avx-regression-fma-alias.S'), diff --git a/tests/integration/vshufpd-interleave.asm b/tests/integration/vshufpd-interleave.asm new file mode 100644 index 0000000000..2efffdac74 --- /dev/null +++ b/tests/integration/vshufpd-interleave.asm @@ -0,0 +1,88 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +section .text + +%macro CASE 3 + %assign narrow 0 + %ifidni %1, xmm + %assign narrow 1 + %endif + vmovdqu ymm0, [junk] + vmovdqu ymm1, [input_a] + vmovdqu ymm2, [input_b] + %if %3 == 7 + vmovdqu xmm1, [input_a] + vmovdqu xmm2, [input_b] + %endif + %if %3 == 0 || %3 == 7 + vshufpd %1 %+ 0, %1 %+ 1, %1 %+ 2, %2 + %elif %3 == 1 + vshufpd %1 %+ 1, %1 %+ 1, %1 %+ 2, %2 + vmovdqu ymm0, ymm1 + %elif %3 == 2 + vshufpd %1 %+ 2, %1 %+ 1, %1 %+ 2, %2 + vmovdqu ymm0, ymm2 + %elif %3 == 3 + vshufpd %1 %+ 0, %1 %+ 1, %1 %+ 1, %2 + %elif %3 == 4 + vshufpd %1 %+ 1, %1 %+ 1, %1 %+ 1, %2 + vmovdqu ymm0, ymm1 + %elif %3 == 5 + vshufpd %1 %+ 0, %1 %+ 1, [input_b], %2 + %else + vshufpd %1 %+ 1, %1 %+ 1, [input_b], %2 + vmovdqu ymm0, ymm1 + %endif + vmovdqu [actual], ymm0 + %assign half 0 + %rep 2 + %assign lane 0 + %rep 2 + %assign index ((%2 >> (half * 2 + lane)) & 1) + %if (narrow && half == 1) || (%3 == 7 && half == 1) + xor eax, eax + %elif lane == 0 || %3 == 3 || %3 == 4 + mov rax, [input_a + half * 16 + index * 8] + %else + mov rax, [input_b + half * 16 + index * 8] + %endif + cmp [actual + half * 16 + lane * 8], rax + jne fail + %assign lane lane + 1 + %endrep + %assign half half + 1 + %endrep +%endmacro + +_start: + mov edi, 1 + %assign immediate 0 + %rep 256 + %assign alias 0 + %rep 8 + CASE ymm, immediate, alias + CASE xmm, immediate, alias + %assign alias alias + 1 + %endrep + %assign immediate immediate + 1 + %endrep + xor edi, edi +fail: + mov eax, 60 + syscall + +section .rodata align=32 +input_a: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0x0123456789abcdef, 0x1020304050607080 +input_b: dq 0x7ff8123456789abc, 0xffffffffffffffff + dq 0xfedcba9876543210, 0x8877665544332211 +junk: times 4 dq 0xdeadbeefdeadbeef + +section .bss align=32 +actual: resb 32 + +section .note.GNU-stack noalloc noexec nowrite progbits From d10a5b7c67e6e04c4ae1cff888b7d8cd6a369996 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Wed, 4 Nov 2026 16:28:33 +0800 Subject: [PATCH 28/33] LATX, opt: Avoid VCVTSI2SD result copies when dest is src1 Insert the converted low double directly into dest when it is src1. Its high double already contains the required source bits, so the result shuffle and full-register copy are unnecessary. Signed-off-by: Zhaoyang Xie --- target/i386/latx/translator/tr-avx-cvt.c | 8 +- .../integration/registrations/avx/meson.build | 1 + tests/integration/vcvtsi2sd-alias.asm | 94 +++++++++++++++++++ 3 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 tests/integration/vcvtsi2sd-alias.asm diff --git a/target/i386/latx/translator/tr-avx-cvt.c b/target/i386/latx/translator/tr-avx-cvt.c index 49fae2cd0f..ce06913712 100644 --- a/target/i386/latx/translator/tr-avx-cvt.c +++ b/target/i386/latx/translator/tr-avx-cvt.c @@ -616,9 +616,11 @@ bool translate_vcvtsi2sd(IR1_INST * pir1) { } else { lsassert(0); } - la_vshuf4i_d(result, src1, 0xc); - if (!ir2_opnd_cmp(&result, &dest)) - la_xvori_b(dest, result, 0); + if (ir2_opnd_cmp(&dest, &src1)) { + la_xvinsve0_d(dest, result, 0); + } else { + la_vshuf4i_d(dest, src1, 0xc); + } set_high128_xreg_to_zero(dest); return true; } diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index 65428276a0..c0ef472b30 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -132,6 +132,7 @@ if 'x86_64-linux-user' in target_dirs files('../../vroundss-fixed.asm'), files('../../vcvtss2sd-alias.asm'), files('../../vcvtdq2pd-ymm-alias.asm'), + files('../../vcvtsi2sd-alias.asm'), files('../../vshufpd-interleave.asm'), files('../../avx-regression-extract-alias.S'), files('../../avx-regression-narrow-state.S'), diff --git a/tests/integration/vcvtsi2sd-alias.asm b/tests/integration/vcvtsi2sd-alias.asm new file mode 100644 index 0000000000..d5d48ff847 --- /dev/null +++ b/tests/integration/vcvtsi2sd-alias.asm @@ -0,0 +1,94 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro CHECK_RESULT 1 + vmovdqu [actual], ymm0 + mov rax, [actual] + mov rdx, %1 + cmp rax, rdx + jne fail + mov rax, [actual + 8] + cmp rax, [initial + 8] + jne fail + mov rax, [actual + 16] + or rax, [actual + 24] + jne fail +%endmacro + +%macro PREPARE 1 + vmovdqu ymm0, [initial] + vmovdqu ymm1, [initial] +%if %1 + vmovdqu ymm2, [initial] +%endif +%endmacro + +_start: + PREPARE 1 + mov eax, [src32] + vcvtsi2sd xmm0, xmm0, eax + CHECK_RESULT 0xc1e0000000000000 + + PREPARE 1 + mov eax, [src32 + 4] + vcvtsi2sd xmm0, xmm0, eax + CHECK_RESULT 0x41dfffffffc00000 + + PREPARE 1 + mov eax, [src32 + 8] + vcvtsi2sd xmm0, xmm0, eax + CHECK_RESULT 0xbff0000000000000 + + PREPARE 1 + mov rax, [src64] + vcvtsi2sd xmm0, xmm0, rax + CHECK_RESULT 0x4270000000001000 + + PREPARE 1 + mov rax, [src64 + 8] + vcvtsi2sd xmm0, xmm0, rax + CHECK_RESULT 0xc270000000001000 + + PREPARE 1 + mov eax, [src32] + vcvtsi2sd xmm0, xmm1, eax + CHECK_RESULT 0xc1e0000000000000 + + PREPARE 1 + mov rax, [src64] + vcvtsi2sd xmm0, xmm1, rax + CHECK_RESULT 0x4270000000001000 + + PREPARE 0 + vcvtsi2sd xmm0, xmm0, dword [src32 + 4] + CHECK_RESULT 0x41dfffffffc00000 + + PREPARE 0 + vcvtsi2sd xmm0, xmm0, qword [src64 + 8] + CHECK_RESULT 0xc270000000001000 + + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +initial: + dq 0x0123456789abcdef, 0x1122334455667788 + dq 0xdeadbeefdeadbeef, 0xfedcba9876543210 +src32: + dd 0x80000000, 0x7fffffff, -1 +src64: + dq 0x0000010000000001, -0x0000010000000001 + +section .bss align=32 +actual: resb 32 + +section .note.GNU-stack noalloc noexec nowrite progbits From dbd2d3fc4640647c24afb04dfb703ee0f853e45e Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 9 Nov 2026 16:42:40 +0800 Subject: [PATCH 29/33] LATX, opt: Skip unused src1 upper-half clears in VINSERTI128 Apply the VINSERTF128 high-insert handling to VINSERTI128. A high insert replaces src1's upper half, so read only its low half and skip its pending upper-half clear. Keep src2 materialization and low-insert handling. --- target/i386/latx/translator/tr-avx.c | 6 +-- target/i386/latx/translator/translate.c | 4 +- .../integration/vinsertf128-high-pending.asm | 47 ++++++++++--------- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 5a39d821e5..dcb41e143f 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -1529,12 +1529,10 @@ bool translate_vinserti128(IR1_INST * pir1) { ir1_opnd_size(ir1_get_opnd(pir1, 2)) == 128 && ir1_opnd_is_imm(ir1_get_opnd(pir1, 3))); uint8 imm = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)) & 0x1; - bool high_insert_partial_read = imm && - ir1_opcode(pir1) == dt_X86_INS_VINSERTF128; - IR2_OPND dest = high_insert_partial_read ? + IR2_OPND dest = imm ? ra_alloc_xmm(ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0))) : load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); - IR2_OPND src1 = high_insert_partial_read ? + IR2_OPND src1 = imm ? ra_alloc_xmm(ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) : load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); diff --git a/target/i386/latx/translator/translate.c b/target/i386/latx/translator/translate.c index e665807cb1..5b0fde9538 100644 --- a/target/i386/latx/translator/translate.c +++ b/target/i386/latx/translator/translate.c @@ -1926,7 +1926,9 @@ bool ir1_translate(IR1_INST *ir1) if (ir1_opnd_is_ymm(opnd)) { /* A high insert reads only src1's low 128 bits. */ - if (i == 1 && ir1_opcode(ir1) == dt_X86_INS_VINSERTF128 && + if (i == 1 && + (ir1_opcode(ir1) == dt_X86_INS_VINSERTF128 || + ir1_opcode(ir1) == dt_X86_INS_VINSERTI128) && (ir1_opnd_uimm(ir1_get_opnd(ir1, 3)) & 1)) { continue; } diff --git a/tests/integration/vinsertf128-high-pending.asm b/tests/integration/vinsertf128-high-pending.asm index 9139c8dbab..99a756caab 100644 --- a/tests/integration/vinsertf128-high-pending.asm +++ b/tests/integration/vinsertf128-high-pending.asm @@ -18,29 +18,29 @@ global _start loop %%next %endmacro -%macro INSERT 2 +%macro INSERT 3 vmovdqu ymm0, [input_b] vmovdqu ymm1, [input_a] vmovdqu ymm2, [input_b] vmovdqa xmm1, xmm1 vmovdqa xmm0, xmm0 %if %2 = 0 - vinsertf128 ymm0, ymm1, xmm2, %1 + %3 ymm0, ymm1, xmm2, %1 %elif %2 = 1 vmovdqa xmm0, xmm1 - vinsertf128 ymm0, ymm0, xmm2, %1 + %3 ymm0, ymm0, xmm2, %1 %elif %2 = 2 - vinsertf128 ymm0, ymm1, xmm0, %1 + %3 ymm0, ymm1, xmm0, %1 %elif %2 = 3 - vinsertf128 ymm0, ymm1, xmm1, %1 + %3 ymm0, ymm1, xmm1, %1 %elif %2 = 4 vmovdqa xmm0, xmm1 - vinsertf128 ymm0, ymm0, xmm0, %1 + %3 ymm0, ymm0, xmm0, %1 %elif %2 = 5 - vinsertf128 ymm0, ymm1, [input_b], %1 + %3 ymm0, ymm1, [input_b], %1 %else vmovdqa xmm0, xmm1 - vinsertf128 ymm0, ymm0, [input_b], %1 + %3 ymm0, ymm0, [input_b], %1 %endif %if %2 = 3 || %2 = 4 %if %1 & 1 @@ -58,18 +58,18 @@ global _start CHECK ymm1, expected_a_zero %endmacro -; %1 selects distinct, src1 alias or src2 alias. Read exactly 16 bytes. -%macro MEMORY 2 +; %1 selects distinct registers or dest == src1. Read exactly 16 bytes. +%macro MEMORY 3 vmovdqu ymm0, [input_b] vmovdqu ymm1, [input_a] lea rdi, [r13 - %2] vmovdqa xmm0, xmm0 vmovdqa xmm1, xmm1 %if %1 = 1 - vinsertf128 ymm0, ymm0, [rdi], 255 + %3 ymm0, ymm0, [rdi], 255 CHECK ymm0, expected_bb_high %else - vinsertf128 ymm0, ymm1, [rdi], 255 + %3 ymm0, ymm1, [rdi], 255 CHECK ymm0, expected_ab_high %endif CHECK ymm1, expected_a_zero @@ -82,7 +82,8 @@ _start: %rep 256 %assign alias 0 %rep 7 - INSERT imm, alias + INSERT imm, alias, vinsertf128 + INSERT imm, alias, vinserti128 %assign alias alias + 1 %endrep %assign imm imm + 1 @@ -115,8 +116,10 @@ _start: syscall test eax, eax jne fail - MEMORY 0, 16 - MEMORY 1, 16 + MEMORY 0, 16, vinsertf128 + MEMORY 1, 16, vinsertf128 + MEMORY 0, 16, vinserti128 + MEMORY 1, 16, vinserti128 mov eax, 13 mov edi, 11 @@ -126,8 +129,7 @@ _start: syscall test eax, eax jne fail -%assign alias 0 -%rep 2 +%macro CROSS_PAGE 2 ; Reprotect before each cross-page load, then recreate deferred clears. mov eax, 10 mov rdi, r13 @@ -145,10 +147,13 @@ _start: syscall test eax, eax jne fail - MEMORY alias, 15 -%assign alias alias + 1 -%endrep - cmp dword [faults], 2 + MEMORY %1, 15, %2 +%endmacro + CROSS_PAGE 0, vinsertf128 + CROSS_PAGE 1, vinsertf128 + CROSS_PAGE 0, vinserti128 + CROSS_PAGE 1, vinserti128 + cmp dword [faults], 4 jne fail mov eax, 11 mov rdi, r12 From eb2f63e7e6f3c108d16a47289834793ee722147f Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Wed, 11 Nov 2026 09:41:55 +0800 Subject: [PATCH 30/33] LATX, fix: Keep memory SETA outside scalar compare fusion A fused UCOMISD/SETA store uses the comparison's instruction boundary. If the store faults, the signal context reports the comparison PC instead of the SETA PC. Translate memory SETA separately and retain all comparison flags for its fault context. --- target/i386/latx/optimization/insts-pattern.c | 5 + .../integration/registrations/avx/meson.build | 7 + tests/integration/ucomisd-seta-fault.asm | 208 ++++++++++++++++++ 3 files changed, 220 insertions(+) create mode 100644 tests/integration/ucomisd-seta-fault.asm diff --git a/target/i386/latx/optimization/insts-pattern.c b/target/i386/latx/optimization/insts-pattern.c index 6c1d5c915d..71d7aeb103 100644 --- a/target/i386/latx/optimization/insts-pattern.c +++ b/target/i386/latx/optimization/insts-pattern.c @@ -339,6 +339,11 @@ static int inst_pattern(TranslationBlock *tb, ir1 = SCAN_IR1(tb, scan, 0); switch (ir1_opcode(ir1)) { case WRAP(SETA): + /* A fused store has no separate SETA fault boundary. */ + if (!ir1_opnd_is_gpr(ir1_get_opnd(ir1, 0))) { + ir1_set_eflag_def(pir1, 0x3f); + return 0; + } pir1->instptn.opc = INSTPTN_OPC_UCOMISD_SETA; pir1->instptn.next = ir1; ir1->instptn.opc = INSTPTN_OPC_NOP; diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index c0ef472b30..e26aa37574 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -61,6 +61,13 @@ if 'x86_64-linux-user' in target_dirs 'args': [emulators['latx-x86_64'], files('../../vcomis-jcc.S')], 'timeout': 120, }] + latx_integration_tests += [{ + 'name': 'test-ucomisd-seta-fault', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], files('../../ucomisd-seta-fault.asm')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 180, + }] latx_integration_tests += [{ 'name': 'test-vcomis-flag-reduction', 'runner': find_program('../../test-avx-regressions.py'), diff --git a/tests/integration/ucomisd-seta-fault.asm b/tests/integration/ucomisd-seta-fault.asm new file mode 100644 index 0000000000..310331c163 --- /dev/null +++ b/tests/integration/ucomisd-seta-fault.asm @@ -0,0 +1,208 @@ +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +%macro ARM 2 + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + xor edx, edx + syscall + test eax, eax + jne fail_setup + lea rax, [%1] + mov [expected_pc], rax + mov qword [expected_flags], %2 + mov dword [seen], 0 + vmovdqu ymm0, [two] + vmovdqu ymm1, [one] + mov r8, 0x1122334455667788 + push qword 0xad7 + popfq +%endmacro + +%macro DONE 0 + cmp dword [seen], 1 + jne fail_count + cmp r8b, 1 + jne fail_result + vmovdqu [actual], ymm0 + vmovdqu [actual + 32], ymm1 +%assign i 0 +%rep 4 + mov rax, [actual + i * 8] + cmp rax, [two + i * 8] + jne fail_state + mov rax, [actual + 32 + i * 8] + cmp rax, [one + i * 8] + jne fail_state +%assign i i + 1 +%endrep +%endmacro + +%macro TESTS 1 + ; A fault loading the comparison source must retain pre-comparison flags. + mov dword [stage], 1 + ARM %%load, 0x8d5 +%%load: + %1 xmm0, [r13] + seta r8b + xor ecx, ecx + DONE + + ; A later fault must expose comparison flags, even if XOR kills them. + mov dword [stage], 2 + ARM %%after, 0 + %1 xmm0, xmm1 + seta r8b +%%after: + mov rax, [r13] + xor ecx, ecx + DONE + + ; A memory SETA needs the new flags and the SETA PC on a store fault. + mov dword [stage], 3 + ARM %%store, 0 + %1 xmm0, xmm1 +%%store: + seta byte [r13] + mov r8b, [r13] + xor ecx, ecx + DONE + + ; The source operand is only eight bytes, ending at a protected page. + ARM %%unused, 0 + %1 xmm0, [r13 - 8] + seta r8b + xor ecx, ecx + cmp dword [seen], 0 + jne fail_count + cmp r8b, 1 + jne fail_result + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + mov edx, 3 + syscall +%%unused: +%endmacro + +section .text +_start: + ldmxcsr [clean] + mov eax, 13 + mov edi, 11 + lea rsi, [action] + xor edx, edx + mov r10d, 8 + syscall + test eax, eax + jne fail_setup + mov eax, 9 + xor edi, edi + mov esi, 131072 + mov edx, 3 + mov r10d, 0x22 + mov r8, -1 + xor r9d, r9d + syscall + cmp rax, -4095 + jae fail_setup + mov r12, rax + lea r13, [rax + 65536] + mov rax, [one] + mov [r13], rax + mov [r13 - 8], rax + TESTS ucomisd + TESTS vucomisd + mov eax, 11 + mov rdi, r12 + mov esi, 131072 + syscall + xor edi, edi + jmp exit + +handler: + cmp edi, 11 + jne fail_signal + cmp dword [seen], 0 + jne fail_count + cmp [rsi + 16], r13 + jne fail_address + mov rax, [rdx + 168] + cmp rax, [expected_pc] + jne fail_pc + mov rax, [rdx + 176] + and eax, 0x8d5 + cmp rax, [expected_flags] + jne fail_flags + mov r9, [rdx + 224] +%assign i 0 +%rep 2 + mov rax, [r9 + 160 + i * 8] + cmp rax, [two + i * 8] + jne fail_state + mov rax, [r9 + 176 + i * 8] + cmp rax, [one + i * 8] + jne fail_state + mov rax, [r9 + 576 + i * 8] + cmp rax, [two + 16 + i * 8] + jne fail_state + mov rax, [r9 + 592 + i * 8] + cmp rax, [one + 16 + i * 8] + jne fail_state +%assign i i + 1 +%endrep + inc dword [seen] + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + mov edx, 3 + syscall + test eax, eax + jne fail_setup + ret +restorer: + mov eax, 15 + syscall + ud2 +fail_setup: mov edi, 1 + jmp exit +fail_signal: mov edi, 2 + jmp exit +fail_count: mov edi, 3 + jmp exit +fail_address: mov edi, 4 + jmp exit +fail_pc: + mov edi, [stage] + imul edi, edi, 10 + add edi, 5 + jmp exit +fail_flags: + mov edi, [stage] + imul edi, edi, 10 + add edi, 6 + jmp exit +fail_state: mov edi, 7 + jmp exit +fail_result: mov edi, 8 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +two: dq 0x4000000000000000, 0x0123456789abcdef + dq 0x1122334455667788, 0xfedcba9876543210 +one: dq 0x3ff0000000000000, 0x9999888877776666 + dq 0xaabbccddeeff0011, 0x123456789abcdef0 +action: dq handler, 0x04000004, restorer, 0 +clean: dd 0x1f80 +section .bss align=32 +actual: resb 64 +expected_pc: resq 1 +expected_flags: resq 1 +seen: resd 1 +stage: resd 1 +section .note.GNU-stack noalloc noexec nowrite progbits From a47b88461fc524c8c20720276667283094d90881 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Wed, 11 Nov 2026 12:06:43 +0800 Subject: [PATCH 31/33] LATX, fix: Preserve comparison flags before faults after VUCOMISD/SETA SETA reads only CF and ZF, but a later page fault can expose every comparison flag to a signal handler. Keep the full flag write for adjacent VUCOMISD and SETA even when ordinary flag analysis marks other flags dead. --- target/i386/latx/optimization/insts-pattern.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/target/i386/latx/optimization/insts-pattern.c b/target/i386/latx/optimization/insts-pattern.c index 71d7aeb103..2c120e1b0f 100644 --- a/target/i386/latx/optimization/insts-pattern.c +++ b/target/i386/latx/optimization/insts-pattern.c @@ -48,6 +48,9 @@ static inline bool ir1_can_pattern(IR1_INST *pir1) case WRAP(CDQ): case WRAP(TEST): case WRAP(UCOMISD): +#ifdef CONFIG_LATX_AVX_OPT + case WRAP(VUCOMISD): +#endif #ifdef CONFIG_LATX_SMC_OPT case WRAP(MOVAPS): case WRAP(MOVDQA): @@ -101,6 +104,9 @@ static inline bool ir1_is_pattern_head(IR1_INST *pir1) case WRAP(CDQ): case WRAP(TEST): case WRAP(UCOMISD): +#ifdef CONFIG_LATX_AVX_OPT + case WRAP(VUCOMISD): +#endif #ifdef CONFIG_LATX_SMC_OPT case WRAP(MOVAPS): case WRAP(MOVDQA): @@ -333,12 +339,24 @@ static int inst_pattern(TranslationBlock *tb, return 0; } case WRAP(UCOMISD): +#ifdef CONFIG_LATX_AVX_OPT + case WRAP(VUCOMISD): +#endif SCAN_CHECK(scan, 0); instptn_check_ucomisd_seta_0(); ir1 = SCAN_IR1(tb, scan, 0); switch (ir1_opcode(ir1)) { case WRAP(SETA): +#ifdef CONFIG_LATX_AVX_OPT + if (ir1_opcode(pir1) == WRAP(VUCOMISD)) { + /* A later fault must expose all flags of the comparison. */ + if (ir1_addr_next(pir1) == ir1_addr(ir1)) { + ir1_set_eflag_def(pir1, 0x3f); + } + return 0; + } +#endif /* A fused store has no separate SETA fault boundary. */ if (!ir1_opnd_is_gpr(ir1_get_opnd(ir1, 0))) { ir1_set_eflag_def(pir1, 0x3f); From b2d37b7713e404ae040f594704fffd6bee7efa2e Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Wed, 11 Nov 2026 12:10:48 +0800 Subject: [PATCH 32/33] LATX, opt: Share UCOMISD and VUCOMISD register SETA fusion Share the adjacent register SETA path between UCOMISD and VUCOMISD. Use one quiet greater-than comparison when no flags need to survive. Discard flags only when a retained register ADD, SUB or CMP overwrites all of them in the same block with no possible fault in between. Otherwise, keep all flags for faults and successors. Perform the floating comparison before writing flags so an invalid-operation trap sees the previous state. Reuse CF and ZF to compute PF when this removes a floating comparison. --- target/i386/latx/optimization/insts-pattern.c | 12 +- target/i386/latx/translator/tr-pattern.c | 98 ++++++++- .../integration/registrations/avx/meson.build | 25 +++ tests/integration/ucomisd-seta-fpe.asm | 190 ++++++++++++++++++ tests/integration/ucomisd-seta-i386.asm | 115 +++++++++++ tests/integration/ucomisd-seta.asm | 185 +++++++++++++++++ 6 files changed, 606 insertions(+), 19 deletions(-) create mode 100644 tests/integration/ucomisd-seta-fpe.asm create mode 100644 tests/integration/ucomisd-seta-i386.asm create mode 100644 tests/integration/ucomisd-seta.asm diff --git a/target/i386/latx/optimization/insts-pattern.c b/target/i386/latx/optimization/insts-pattern.c index 2c120e1b0f..b20f3b209d 100644 --- a/target/i386/latx/optimization/insts-pattern.c +++ b/target/i386/latx/optimization/insts-pattern.c @@ -346,17 +346,11 @@ static int inst_pattern(TranslationBlock *tb, instptn_check_ucomisd_seta_0(); ir1 = SCAN_IR1(tb, scan, 0); + if (ir1_addr_next(pir1) != ir1_addr(ir1)) { + return 0; + } switch (ir1_opcode(ir1)) { case WRAP(SETA): -#ifdef CONFIG_LATX_AVX_OPT - if (ir1_opcode(pir1) == WRAP(VUCOMISD)) { - /* A later fault must expose all flags of the comparison. */ - if (ir1_addr_next(pir1) == ir1_addr(ir1)) { - ir1_set_eflag_def(pir1, 0x3f); - } - return 0; - } -#endif /* A fused store has no separate SETA fault boundary. */ if (!ir1_opnd_is_gpr(ir1_get_opnd(ir1, 0))) { ir1_set_eflag_def(pir1, 0x3f); diff --git a/target/i386/latx/translator/tr-pattern.c b/target/i386/latx/translator/tr-pattern.c index 9fc717eb45..9ddcc01113 100644 --- a/target/i386/latx/translator/tr-pattern.c +++ b/target/i386/latx/translator/tr-pattern.c @@ -1395,6 +1395,69 @@ static bool translate_test_xxcc(IR1_INST *pir1) return true; } +#ifdef CONFIG_LATX_XCOMISX_OPT +static uint8_t ucomisd_seta_live_flags(IR1_INST *next) +{ +#ifdef CONFIG_LATX_FLAG_REDUCTION + if (option_flag_reduction && + ir1_opnd_is_gpr(ir1_get_opnd(next, 0))) { + TranslationBlock *tb = lsenv->tr_data->curr_tb; + uint8_t flags = 0; + + /* Keep all flags before a possible fault or an unknown successor. + * Only discard them across non-faulting register operations ending + * in an unconditional flag overwrite within this TB. */ + for (IR1_INST *inst = next + 1; inst <= tb_ir1_inst_last(tb); inst++) { + if (ir1_is_prefix_lock(inst)) { + return 0x3f; + } + for (int i = 0; i < ir1_opnd_num(inst); i++) { + IR1_OPND *opnd = ir1_get_opnd(inst, i); + if (!ir1_opnd_is_gpr(opnd) && !ir1_opnd_is_imm(opnd)) { + return 0x3f; + } + } + switch (ir1_opcode(inst)) { + case WRAP(ADD): + case WRAP(SUB): + case WRAP(CMP): + /* Require a retained, unfused write of every flag. */ + return inst->instptn.opc == INSTPTN_OPC_NONE && + ir1_get_eflag_def(inst) == 0x3f ? flags : 0x3f; + case WRAP(MOV): + case WRAP(MOVZX): + case WRAP(MOVSX): + case WRAP(MOVSXD): + case WRAP(NOP): + case WRAP(LAHF): + case WRAP(SETO): + case WRAP(SETNO): + case WRAP(SETB): + case WRAP(SETAE): + case WRAP(SETE): + case WRAP(SETNE): + case WRAP(SETBE): + case WRAP(SETA): + case WRAP(SETS): + case WRAP(SETNS): + case WRAP(SETP): + case WRAP(SETNP): + case WRAP(SETL): + case WRAP(SETGE): + case WRAP(SETLE): + case WRAP(SETG): + flags |= ir1_get_eflag_use(inst); + break; + default: + return 0x3f; + } + } + } +#endif + return 0x3f; +} +#endif + static bool translate_ucomisd_seta(IR1_INST *pir1) { IR1_INST *curr = pir1; @@ -1404,6 +1467,15 @@ static bool translate_ucomisd_seta(IR1_INST *pir1) IR1_OPND *opnd1 = ir1_get_opnd(curr, 1); IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src = load_freg128_from_ir1(opnd1); +#ifdef CONFIG_LATX_XCOMISX_OPT + uint8_t eflags = ucomisd_seta_live_flags(next); + const uint8_t seta_flags = CF_USEDEF_BIT | ZF_USEDEF_BIT; + /* Two comparisons can supply CF, ZF, PF and SETA together. */ + if ((eflags & PF_USEDEF_BIT) && (eflags & seta_flags)) { + eflags |= seta_flags; + } + IR2_OPND flag = ra_alloc_itemp(); +#else /* 0. set flag = 0 */ IR2_OPND flag_zf = ra_alloc_itemp(); IR2_OPND flag_pf = ra_alloc_itemp(); @@ -1428,19 +1500,25 @@ static bool translate_ucomisd_seta(IR1_INST *pir1) ra_free_temp(flag_zf); /* 4. mov flag to EFLAGS */ la_x86mtflag(flag, 0x3f); +#endif - lsenv->tr_data->curr_ir1_inst = next; - lsenv->tr_data->curr_ir1_count++; - switch (ir1_opcode(next)) { - case WRAP(SETA): +#ifdef CONFIG_LATX_XCOMISX_OPT + if ((eflags & seta_flags) == seta_flags) { + generate_xcomisx(dest, src, true, false, eflags); + la_setx86j(flag, COND_A); + } else +#endif + { + lsassert(ir1_opcode(next) == WRAP(SETA)); la_fcmp_cond_d(fcc0_ir2_opnd, src, dest, FCMP_COND_CLT); - break; - default: - lsassert(0); - break; + la_movcf2gr(flag, fcc0_ir2_opnd); +#ifdef CONFIG_LATX_XCOMISX_OPT + /* The compare can trap before even OF/AF/SF may be cleared. */ + generate_xcomisx(dest, src, true, false, eflags); +#endif } - - la_movcf2gr(flag, fcc0_ir2_opnd); + lsenv->tr_data->curr_ir1_inst = next; + lsenv->tr_data->curr_ir1_count++; store_ireg_to_ir1(flag, ir1_get_opnd(next, 0), false); ra_free_temp(flag); diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index e26aa37574..6860f645c0 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -1,3 +1,14 @@ +if 'i386-linux-user' in target_dirs + latx_integration_tests += [{ + 'name': 'test-ucomisd-seta-i386', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-i386'], '--guest-bits', '32', + files('../../ucomisd-seta-i386.asm')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 180, + }] +endif + if 'x86_64-linux-user' in target_dirs foreach cvt_option : ['0', '1'] latx_integration_tests += [{ @@ -61,6 +72,13 @@ if 'x86_64-linux-user' in target_dirs 'args': [emulators['latx-x86_64'], files('../../vcomis-jcc.S')], 'timeout': 120, }] + latx_integration_tests += [{ + 'name': 'test-ucomisd-seta', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], files('../../ucomisd-seta.asm')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 180, + }] latx_integration_tests += [{ 'name': 'test-ucomisd-seta-fault', 'runner': find_program('../../test-avx-regressions.py'), @@ -68,6 +86,13 @@ if 'x86_64-linux-user' in target_dirs 'env': {'LATX_SOFTFPU': '0'}, 'timeout': 180, }] + latx_integration_tests += [{ + 'name': 'test-ucomisd-seta-fpe', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], files('../../ucomisd-seta-fpe.asm')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 180, + }] latx_integration_tests += [{ 'name': 'test-vcomis-flag-reduction', 'runner': find_program('../../test-avx-regressions.py'), diff --git a/tests/integration/ucomisd-seta-fpe.asm b/tests/integration/ucomisd-seta-fpe.asm new file mode 100644 index 0000000000..47d6a6ae37 --- /dev/null +++ b/tests/integration/ucomisd-seta-fpe.asm @@ -0,0 +1,190 @@ +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +%macro CASE 1 + mov dword [seen], 0 + mov dword [expected_set], 0x40 + lea rax, [%%trap] + mov [expected_pc], rax + lea rax, [%%resume] + mov [resume_pc], rax + movsd xmm0, [one] + movsd xmm1, [snan] + mov eax, 1 + test eax, eax + pushfq + pop rdi + %1 xmm0, xmm0 + seta al + setc r8b + ; This flag write can be removed because the next comparison defines ZF. + xor ecx, ecx +%%trap: + %1 xmm1, xmm0 +%%resume: + setz r9b + xor ecx, ecx + cmp r9b, 1 + jne fail_flags + cmp r8b, 0 + jne fail_result + cmp dword [seen], 1 + jne fail_count +%endmacro + +%macro ONLY_OVERFLOW_LIVE 1 + mov dword [seen], 0 + mov dword [expected_set], 0x8d5 + lea rax, [%%trap] + mov [expected_pc], rax + lea rax, [%%resume] + mov [resume_pc], rax + movsd xmm0, [one] + movsd xmm1, [snan] + push qword 0x8d7 + popfq + jmp %%trap +%%trap: + %1 xmm1, xmm0 +%%resume: + seta al + seto r8b + cmp ecx, ecx + pushfq + pop rcx + test al, al + jne fail_result + cmp r8b, 1 + jne fail_flags + cmp dword [seen], 1 + jne fail_count +%endmacro + +%macro SNAN_MASK 3 + mov dword [seen], 0 + mov dword [expected_set], 0x8d5 + lea rax, [%%trap] + mov [expected_pc], rax + lea rax, [%%resume] + mov [resume_pc], rax + ldmxcsr [mxcsr] + movsd xmm0, [snan] + movsd xmm1, [one] + push qword 0x8d7 + popfq + jmp %%trap +%%trap: +%if %2 = 0 + %1 xmm0, xmm1 +%elif %2 = 1 + %1 xmm0, xmm0 +%else + %1 xmm0, [one] +%endif +%%resume: + seta al +%if %3 & 1 + setc r8b +%endif +%if %3 & 2 + setp r9b +%endif +%if %3 & 4 + lahf +%endif +%if %3 & 8 + setz r10b +%endif +%if %3 & 16 + sets r11b +%endif +%if %3 & 32 + seto r12b +%endif + cmp ecx, ecx + pushfq + pop rcx + test al, al + jne fail_result + cmp dword [seen], 1 + jne fail_count +%endmacro + +section .text +_start: + mov eax, 13 + mov edi, 8 + lea rsi, [action] + xor edx, edx + mov r10d, 8 + syscall + test eax, eax + jne fail_setup + ; Both x87 and SSE invalid-operation traps are enabled, not soft float. + fldcw [control] + ldmxcsr [mxcsr] + CASE ucomisd + CASE vucomisd + ONLY_OVERFLOW_LIVE ucomisd + ONLY_OVERFLOW_LIVE vucomisd +%assign mask 0 +%rep 64 +%assign form 0 +%rep 3 + SNAN_MASK ucomisd, form, mask + SNAN_MASK vucomisd, form, mask +%assign form form + 1 +%endrep +%assign mask mask + 1 +%endrep + xor edi, edi + jmp exit +handler: + cmp edi, 8 + jne fail_signal + mov rax, [rdx + 168] + cmp rax, [expected_pc] + jne fail_pc + mov eax, [rdx + 176] + and eax, [expected_set] + cmp eax, [expected_set] + jne fail_flags + cmp dword [seen], 0 + jne fail_count + inc dword [seen] + ; Skip the deliberate fault, as the Lative random-test handler does. + mov rax, [resume_pc] + mov [rdx + 168], rax + ret +restorer: + mov eax, 15 + syscall + ud2 +fail_setup: mov edi, 1 + jmp exit +fail_signal: mov edi, 2 + jmp exit +fail_pc: mov edi, 3 + jmp exit +fail_flags: mov edi, 4 + jmp exit +fail_result: mov edi, 5 + jmp exit +fail_count: mov edi, 6 +exit: + mov eax, 60 + syscall +section .rodata +one: dq 0x3ff0000000000000 +snan: dq 0x7ff0000000000001 +control: dw 0x037e +mxcsr: dd 0x1f00 +action: dq handler, 0x04000004, restorer, 0 +section .bss +expected_pc: resq 1 +resume_pc: resq 1 +seen: resd 1 +expected_set: resd 1 +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/ucomisd-seta-i386.asm b/tests/integration/ucomisd-seta-i386.asm new file mode 100644 index 0000000000..136c82908c --- /dev/null +++ b/tests/integration/ucomisd-seta-i386.asm @@ -0,0 +1,115 @@ +; SPDX-License-Identifier: GPL-2.0-only +bits 32 +global _start + +%macro CASE 6 + mov eax, 0x12345678 + mov ebx, eax + mov ecx, eax + mov edx, eax +%if %2 = 0 + %1 xmm0, xmm1 +%elif %2 = 1 + %1 xmm0, xmm0 +%else + %1 xmm0, [esi + 8] +%endif + seta %3 + mov ebp, %4 +%if %6 + pushfd + pop edx +%else + ; Keep this full flag write so the compare flags are not needed. + cmp ebp, ebp + pushfd + pop edx +%endif + mov edi, [esi + 16] +%if %2 = 1 + mov edi, [esi + 20] +%endif +%if %6 + and edx, 0x8d5 + cmp edx, edi + jne fail_flags +%endif + mov eax, edi + and eax, 0x41 + sete al + movzx eax, al +%if %5 = 8 + shl eax, 8 + or eax, 0x12340078 +%else + or eax, 0x12345600 +%endif + cmp ebp, eax + jne fail_result +%endmacro + +%macro DESTINATIONS 3 + CASE %1, %2, al, eax, 0, %3 + CASE %1, %2, ah, eax, 8, %3 + CASE %1, %2, bl, ebx, 0, %3 + CASE %1, %2, bh, ebx, 8, %3 + CASE %1, %2, cl, ecx, 0, %3 + CASE %1, %2, ch, ecx, 8, %3 + CASE %1, %2, dl, edx, 0, %3 + CASE %1, %2, dh, edx, 8, %3 +%endmacro + +section .text +_start: + ldmxcsr [clean] + mov esi, pairs +next_pair: + movsd xmm0, [esi] + movsd xmm1, [esi + 8] + call suite + add esi, 24 + cmp esi, pairs_end + jne next_pair + xor ebx, ebx + jmp exit +suite: +%assign live 0 +%rep 2 +%assign form 0 +%rep 3 + DESTINATIONS ucomisd, form, live + DESTINATIONS vucomisd, form, live +%assign form form + 1 +%endrep +%assign live live + 1 +%endrep + ret +fail_result: mov ebx, 1 + jmp exit +fail_flags: mov ebx, 2 +exit: + mov eax, 1 + int 0x80 + +%macro PAIR 4 + dq %1, %2 + dd %3, %4 +%endmacro +section .rodata align=8 +clean: dd 0x1f80 +align 8 +pairs: + PAIR 0x4000000000000000, 0x3ff0000000000000, 0, 0x40 + PAIR 0x3ff0000000000000, 0x4000000000000000, 1, 0x40 + PAIR 0x3ff0000000000000, 0x3ff0000000000000, 0x40, 0x40 + PAIR 0, 0x8000000000000000, 0x40, 0x40 + PAIR 0x7ff0000000000000, 0x4000000000000000, 0, 0x40 + PAIR 0xfff0000000000000, 0xbff0000000000000, 1, 0x40 + PAIR 0x7ff8000000000001, 0x3ff0000000000000, 0x45, 0x45 + PAIR 0x3ff0000000000000, 0xfff800000000cafe, 0x45, 0x40 + PAIR 0x7ff0000000000001, 0x3ff0000000000000, 0x45, 0x45 + PAIR 0x3ff0000000000000, 0xfff0000000000001, 0x45, 0x40 + PAIR 1, 0, 0, 0x40 + PAIR 0x8000000000000001, 0, 1, 0x40 +pairs_end: +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/ucomisd-seta.asm b/tests/integration/ucomisd-seta.asm new file mode 100644 index 0000000000..faaba6097a --- /dev/null +++ b/tests/integration/ucomisd-seta.asm @@ -0,0 +1,185 @@ +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +%macro CHECK_FLAG 4 +%if %4 & (1 << %1) + mov eax, ebp + shr eax, %3 + and eax, 1 + cmp %2, al + jne fail_flags +%endif +%endmacro + +%macro CASE 4 + vmovdqu ymm0, [rsi] + vmovdqu ymm1, [rsi + 32] + mov ebp, [rsi + 64] +%if %2 = 1 + mov ebp, [rsi + 68] +%endif + mov rax, 0x123456789abcde00 + mov r8, rax +%if %2 = 0 + %1 xmm0, xmm1 +%elif %2 = 1 + %1 xmm0, xmm0 +%else + %1 xmm0, [rsi + 32] +%endif +%if %3 = 0 + seta al + mov rdx, rax +%elif %3 = 1 + seta ah + mov rdx, rax +%else + seta r8b + mov rdx, r8 +%endif +%if %4 & 1 + setc r9b +%endif +%if %4 & 2 + setp r10b +%endif +%if %4 & 4 + lahf + movzx ebx, ah + mov r14d, ebx +%endif +%if %4 & 8 + setz r11b +%endif +%if %4 & 16 + sets r12b +%endif +%if %4 & 32 + seto r13b +%endif + cmp ecx, ecx + pushfq + pop rcx + mov ecx, 0 + mov eax, ebp + and eax, 0x41 + sete cl +%if %3 = 1 + shl ecx, 8 + mov rbx, 0x123456789abc0000 +%else + mov rbx, 0x123456789abcde00 +%endif + or rbx, rcx + cmp rdx, rbx + jne fail_result + CHECK_FLAG 0, r9b, 0, %4 + CHECK_FLAG 1, r10b, 2, %4 +%if %4 & 4 + mov eax, ebp + or eax, 2 + cmp r14b, al + jne fail_flags +%endif + CHECK_FLAG 3, r11b, 6, %4 + CHECK_FLAG 4, r12b, 7, %4 + CHECK_FLAG 5, r13b, 11, %4 + vmovdqu [actual], ymm0 + vmovdqu [actual + 32], ymm1 +%assign q 0 +%rep 8 + mov rax, [actual + q * 8] + cmp rax, [rsi + q * 8] + jne fail_vector +%assign q q + 1 +%endrep +%endmacro + +%macro STACK_BYTE 5 + mov rdi, %3 + mov rdx, rdi + and rdx, -256 + %1 xmm0, %4 + seta %2 + mov rcx, %3 + mov %3, rdi + cmp eax, eax + pushfq + pop rax + or rdx, %5 + cmp rcx, rdx + jne fail_result +%endmacro + +section .text +_start: + ldmxcsr [clean] + lea rsi, [pairs] + mov r15d, (pairs_end - pairs) / 96 +next_pair: + call suite + add rsi, 96 + dec r15d + jnz next_pair + ; The random-test harness reserves RSP and RBP. + movsd xmm0, [pairs] + movsd xmm1, [pairs + 32] + STACK_BYTE ucomisd, spl, rsp, xmm1, 1 + STACK_BYTE ucomisd, bpl, rbp, xmm0, 0 + STACK_BYTE vucomisd, spl, rsp, xmm0, 0 + STACK_BYTE vucomisd, bpl, rbp, xmm1, 1 + xor edi, edi + jmp exit +suite: +%assign mask 0 +%rep 64 +%assign form 0 +%rep 3 +%assign dest 0 +%rep 3 + CASE ucomisd, form, dest, mask + CASE vucomisd, form, dest, mask +%assign dest dest + 1 +%endrep +%assign form form + 1 +%endrep +%assign mask mask + 1 +%endrep + ret +fail_result: mov edi, 1 + jmp exit +fail_flags: mov edi, 2 + jmp exit +fail_vector: mov edi, 3 +exit: + mov eax, 60 + syscall + +%macro PAIR 4 + dq %1, 0x0123456789abcdef, 0xffeeddccbbaa9988, 0x1122334455667788 + dq %2, 0xfedcba9876543210, 0xaabbccddeeff0011, 0x8877665544332211 + dd %3, %4 + times 24 db 0 +%endmacro +section .rodata align=32 +clean: dd 0x1f80 +align 32 +pairs: + PAIR 0x4000000000000000, 0x3ff0000000000000, 0, 0x40 + PAIR 0x3ff0000000000000, 0x4000000000000000, 1, 0x40 + PAIR 0x3ff0000000000000, 0x3ff0000000000000, 0x40, 0x40 + PAIR 0, 0x8000000000000000, 0x40, 0x40 + PAIR 0x7ff0000000000000, 0x4000000000000000, 0, 0x40 + PAIR 0xfff0000000000000, 0xbff0000000000000, 1, 0x40 + PAIR 0x7ff8000000000001, 0x3ff0000000000000, 0x45, 0x45 + PAIR 0x3ff0000000000000, 0xfff800000000cafe, 0x45, 0x40 + PAIR 0x7ff0000000000001, 0x3ff0000000000000, 0x45, 0x45 + PAIR 0x3ff0000000000000, 0xfff0000000000001, 0x45, 0x40 + PAIR 1, 0, 0, 0x40 + PAIR 0x8000000000000001, 0, 1, 0x40 +pairs_end: +section .bss align=32 +actual: resb 64 +section .note.GNU-stack noalloc noexec nowrite progbits From 50a1bc412893ecc6c641b0fc993dc1b7b596effb Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Wed, 11 Nov 2026 16:35:12 +0800 Subject: [PATCH 33/33] LATX, opt: Write non-aliasing VCMPSD results directly When dest differs from src1, write the comparison mask directly to dest and merge src1's high double there. This removes a temporary result and its copy. Keep the temporary when dest is src1 because its high double is still needed after the comparison. --- target/i386/latx/translator/tr-avx-cmp.c | 421 +++++++++++------- .../integration/registrations/avx/meson.build | 7 + tests/integration/vcmpsd-writeback.asm | 171 +++++++ 3 files changed, 437 insertions(+), 162 deletions(-) create mode 100644 tests/integration/vcmpsd-writeback.asm diff --git a/target/i386/latx/translator/tr-avx-cmp.c b/target/i386/latx/translator/tr-avx-cmp.c index d1e1e0dca8..c795aca14c 100644 --- a/target/i386/latx/translator/tr-avx-cmp.c +++ b/target/i386/latx/translator/tr-avx-cmp.c @@ -1913,6 +1913,7 @@ bool translate_vcmpps(IR1_INST * pir1) { return true; } +/* Scalar double compares read src1's high lane after writing the mask. */ bool translate_vcmpeqsd(IR1_INST * pir1) { if (!option_enable_lasx) { return translate_vcmpsd_lsx(pir1); @@ -1925,17 +1926,20 @@ bool translate_vcmpeqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_EQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_EQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -1951,17 +1955,20 @@ bool translate_vcmpltsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_LT); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_LT); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -1977,17 +1984,20 @@ bool translate_vcmplesd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_LE); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_LE); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2003,17 +2013,20 @@ bool translate_vcmpunordsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_UNORD); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_UNORD); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2029,17 +2042,20 @@ bool translate_vcmpneqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NEQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_NEQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2055,17 +2071,20 @@ bool translate_vcmpnltsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_NLT); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp2, temp1, X86_FCMP_COND_NLT); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2081,17 +2100,20 @@ bool translate_vcmpnlesd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_NLE); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp2, temp1, X86_FCMP_COND_NLE); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2107,17 +2129,20 @@ bool translate_vcmpordsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_ORD); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_ORD); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2133,17 +2158,20 @@ bool translate_vcmpeq_uqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_EQ_UQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_EQ_UQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2159,17 +2187,20 @@ bool translate_vcmpngesd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NGE); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_NGE); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2185,17 +2216,20 @@ bool translate_vcmpngtsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NGT); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_NGT); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2211,17 +2245,20 @@ bool translate_vcmpfalsesd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_FALSE); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_FALSE); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2237,17 +2274,20 @@ bool translate_vcmpneq_oqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NEQ_OQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_NEQ_OQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2263,17 +2303,20 @@ bool translate_vcmpgesd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_GE); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp2, temp1, X86_FCMP_COND_GE); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2289,17 +2332,20 @@ bool translate_vcmpgtsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_GT); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp2, temp1, X86_FCMP_COND_GT); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2315,18 +2361,21 @@ bool translate_vcmptruesd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_TRUE); - la_xvori_b(dest_temp, dest_temp, 0xff); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_TRUE); + la_xvori_b(result, result, 0xff); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2342,17 +2391,20 @@ bool translate_vcmpeq_ossd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_EQ_OS); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_EQ_OS); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2368,17 +2420,20 @@ bool translate_vcmplt_oqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_LT_OQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_LT_OQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2394,17 +2449,20 @@ bool translate_vcmple_oqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_LE_OQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_LE_OQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2420,17 +2478,20 @@ bool translate_vcmpunord_ssd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_UNORD_S); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_UNORD_S); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2446,17 +2507,20 @@ bool translate_vcmpneq_ussd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NEQ_US); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_NEQ_US); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2472,17 +2536,20 @@ bool translate_vcmpnlt_uqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_NLT_UQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp2, temp1, X86_FCMP_COND_NLT_UQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2498,17 +2565,20 @@ bool translate_vcmpnle_uqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_NLE_UQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp2, temp1, X86_FCMP_COND_NLE_UQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2524,17 +2594,20 @@ bool translate_vcmpord_ssd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_ORD_S); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_ORD_S); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2550,17 +2623,20 @@ bool translate_vcmpeq_ussd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_EQ_US); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_EQ_US); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2576,17 +2652,20 @@ bool translate_vcmpnge_uqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NGE_UQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_NGE_UQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2602,17 +2681,20 @@ bool translate_vcmpngt_uqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NGT_UQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_NGT_UQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2628,17 +2710,20 @@ bool translate_vcmpfalse_ossd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_FALSE_OS); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_FALSE_OS); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2654,17 +2739,20 @@ bool translate_vcmpneq_ossd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NEQ_OS); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_NEQ_OS); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2680,17 +2768,20 @@ bool translate_vcmpge_oqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_GE_OQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp2, temp1, X86_FCMP_COND_GE_OQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2706,17 +2797,20 @@ bool translate_vcmpgt_oqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_GT_OQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp2, temp1, X86_FCMP_COND_GT_OQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2732,18 +2826,21 @@ bool translate_vcmptrue_ussd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_TRUE_US); - la_xvori_b(dest_temp, dest_temp, 0xff); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_TRUE_US); + la_xvori_b(result, result, 0xff); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index 6860f645c0..7e56ef9460 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -20,6 +20,13 @@ if 'x86_64-linux-user' in target_dirs }] endforeach if 'CONFIG_LATX_AVX_OPT' in config_host + latx_integration_tests += [{ + 'name': 'test-vcmpsd-writeback', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], files('../../vcmpsd-writeback.asm')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 120, + }] latx_integration_tests += [{ 'name': 'test-f16c-memory-boundary', 'runner': find_program('../../test-avx-regressions.py'), diff --git a/tests/integration/vcmpsd-writeback.asm b/tests/integration/vcmpsd-writeback.asm new file mode 100644 index 0000000000..90a4a76667 --- /dev/null +++ b/tests/integration/vcmpsd-writeback.asm @@ -0,0 +1,171 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%define LESS 0x96969696 +%define EQUAL 0xa5a5a5a5 +%define GREATER 0xf0f0f0f0 +%define UNORDERED 0x87788778 + +%macro VERIFY 2 + vmovdqu [output], %1 + bt %2, (imm & 31) + sbb rdx, rdx + cmp [output], rdx + jne fail + mov rax, [r8 + 8] + cmp [output + 8], rax + jne fail + mov rax, [output + 16] + or rax, [output + 24] + jne fail +%endmacro + +%macro COMPARE 3 + ldmxcsr [masked] + vmovdqu ymm0, [r8] + vmovdqu ymm1, [r8] + vmovdqu ymm2, [r9] + vcmpsd %1, %2, %3, imm + stmxcsr [actual_mxcsr] +%endmacro + +%macro SAME_MXCSR 0 + mov eax, [actual_mxcsr] + cmp eax, [reference_mxcsr] + jne fail +%endmacro + +%macro SAVE_MXCSR 0 + mov eax, [actual_mxcsr] + mov [reference_mxcsr], eax +%endmacro + +%macro CHECK_FINITE_MXCSR 0 + ; A high-lane SNaN must not raise invalid for finite low lanes. + cmp r11d, UNORDERED + je %%unordered + cmp eax, [masked] + jne fail +%%unordered: +%endmacro + +section .text +_start: + ; Put memory operands at the last eight bytes before an inaccessible page. + xor edi, edi + mov esi, 8192 + mov edx, 3 + mov r10d, 0x22 + mov r8, -1 + xor r9d, r9d + mov eax, 9 + syscall + test rax, rax + js fail + lea r13, [rax + 4088] + lea rdi, [rax + 4096] + mov esi, 4096 + xor edx, edx + mov eax, 10 + syscall + test eax, eax + jnz fail + lea r14, [pairs] + lea r15, [pairs_end] +.pair: + mov r8, [r14] + mov r9, [r14 + 8] + mov r11d, [r14 + 16] + mov r12d, [r14 + 20] + mov rax, [r9] + mov [r13], rax + %assign imm 0 + %rep 256 + ; Aliasing src1 retains the original temporary-result path. + COMPARE xmm0, xmm0, xmm2 + VERIFY ymm0, r11d + SAVE_MXCSR + CHECK_FINITE_MXCSR + COMPARE xmm0, xmm1, xmm2 + VERIFY ymm0, r11d + SAME_MXCSR + COMPARE xmm2, xmm1, xmm2 + VERIFY ymm2, r11d + SAME_MXCSR + COMPARE xmm0, xmm0, [r13] + VERIFY ymm0, r11d + SAME_MXCSR + COMPARE xmm0, xmm1, [r13] + VERIFY ymm0, r11d + SAME_MXCSR + + COMPARE xmm1, xmm1, xmm1 + VERIFY ymm1, r12d + SAVE_MXCSR + COMPARE xmm0, xmm1, xmm1 + VERIFY ymm0, r12d + SAME_MXCSR + + ldmxcsr [masked] + vmovdqu ymm8, [r8] + vmovdqu ymm9, [r8] + vmovdqu ymm10, [r9] + vcmpsd xmm8, xmm9, xmm10, imm + VERIFY ymm8, r11d + %assign imm imm + 1 + %endrep + add r14, 24 + cmp r14, r15 + jb .pair + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +one: dq 1.0, 0x7ff0000000000123, -1, -1 +two: dq 2.0, 0x123456789abcdef0, -1, -1 +qnan: dq 0x7ff8000000000001, 0x7ff0000000000456, -1, -1 +snan: dq 0x7ff0000000000001, 0xfedcba9876543210, -1, -1 +zero: dq 0, 0x7ff0000000000123, -1, -1 +negative_zero: dq 0x8000000000000000, 0x0123456789abcdef, -1, -1 +infinity: dq 0x7ff0000000000000, 0x7ff0000000000123, -1, -1 +negative_infinity: dq 0xfff0000000000000, 0x7ff0000000000123, -1, -1 +largest: dq 0x7fefffffffffffff, 0x7ff0000000000123, -1, -1 +pairs: + dq one, two + dd LESS, EQUAL + dq one, one + dd EQUAL, EQUAL + dq two, one + dd GREATER, EQUAL + dq qnan, one + dd UNORDERED, UNORDERED + dq one, snan + dd UNORDERED, EQUAL + dq snan, qnan + dd UNORDERED, UNORDERED + dq zero, negative_zero + dd EQUAL, EQUAL + dq negative_infinity, one + dd LESS, EQUAL + dq infinity, largest + dd GREATER, EQUAL + dq infinity, infinity + dd EQUAL, EQUAL +pairs_end: +masked: dd 0x1f80 + +section .bss align=32 +output: resb 32 +actual_mxcsr: resd 1 +reference_mxcsr: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits