diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index 2ac697c8bb..06d1a55268 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -424,7 +424,6 @@ int encode_search(TranslationBlock *tb, uint8_t *block) #ifdef CONFIG_LATX_OPT_PUSH_POP extern void ir2_opt_push_pop_fix(TranslationBlock *tb, CPUState *cpu, int i); #endif -extern void opt_instptn_fix(CPUState *cpu, TranslationBlock *tb, int i); /* The cpu state corresponding to 'searched_pc' is restored. * When reset_icount is true, current TB will be interrupted and @@ -486,9 +485,6 @@ static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, ir2_opt_push_pop_fix(tb, cpu, (searched_pc - (uintptr_t)tb->tc.ptr) / 4); } #endif -#ifdef CONFIG_LATX_INSTS_PATTERN - opt_instptn_fix(cpu, tb, i); -#endif #ifdef CONFIG_PROFILER qatomic_set(&prof->restore_time, prof->restore_time + profile_getclock() - ti); diff --git a/target/i386/latx/include/insts-pattern.h b/target/i386/latx/include/insts-pattern.h index 781a6677ca..33dc3d8ae4 100644 --- a/target/i386/latx/include/insts-pattern.h +++ b/target/i386/latx/include/insts-pattern.h @@ -163,4 +163,9 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *ir1, int index, bool try_translate_instptn(IR1_INST *pir1); +#ifdef CONFIG_LATX_INSTS_PATTERN +unsigned instptn_flags_recovery(IR1_INST *pir1); +void opt_instptn_fix(CPUArchState *env, unsigned recovery); +#endif + #endif diff --git a/target/i386/latx/include/ir2.h b/target/i386/latx/include/ir2.h index 208d64cfb2..b052475f97 100644 --- a/target/i386/latx/include/ir2.h +++ b/target/i386/latx/include/ir2.h @@ -116,8 +116,5 @@ bool la_ir2_opcode_is_load(IR2_OPCODE opcode); void tr_ir2_optimize(TranslationBlock *tb); void ir2_opt_push_pop_fix(TranslationBlock *tb, CPUState *cpu, int i); -#ifdef CONFIG_LATX_INSTS_PATTERN -void opt_instptn_fix(CPUState *cpu, TranslationBlock *tb, int index); -#endif void la_append_ir2_jmp_far(ADDR jmp_offset); #endif diff --git a/target/i386/latx/include/latx-unwind.h b/target/i386/latx/include/latx-unwind.h new file mode 100644 index 0000000000..baf9dbb5b1 --- /dev/null +++ b/target/i386/latx/include/latx-unwind.h @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: 2021-2026 LAT Project Authors + * + * SPDX-License-Identifier: GPL-2.0-only + */ + +#ifndef LATX_UNWIND_H +#define LATX_UNWIND_H + +/* Layout of the second per-instruction search-table word, also on i386. */ +#define LATX_UNWIND_CC_OP_MASK 0xff +#define LATX_UNWIND_INSTPTN_SHIFT 8 +#define LATX_UNWIND_INSTPTN_MASK 0xff +#define LATX_UNWIND_YMMH_SHIFT 16 + +/* Delayed integer flags use the saved host a0/a1, never temporary IR1. */ +enum { + INSTPTN_FLAGS_NONE, + INSTPTN_FLAGS_CMP, + INSTPTN_FLAGS_TEST, + INSTPTN_FLAGS_BT, +}; + +#define INSTPTN_FLAGS_OP_MASK 3 +#define INSTPTN_FLAGS_SIZE_SHIFT 2 +#define INSTPTN_FLAGS_SIZE_MASK 3 +#define INSTPTN_FLAGS_SAME_REG (1 << 4) + +#endif diff --git a/target/i386/latx/include/translate.h b/target/i386/latx/include/translate.h index 098f05867c..a8859c02f1 100644 --- a/target/i386/latx/include/translate.h +++ b/target/i386/latx/include/translate.h @@ -1119,6 +1119,8 @@ TRANS_FUNC_DEF(vpshufb_lsx); TRANS_FUNC_DEF(vpsubx); TRANS_FUNC_DEF(vptest); TRANS_FUNC_DEF(vptest_lsx); +void generate_avx_test_flags(IR1_INST *, IR2_OPND, IR2_OPND, IR2_OPND, + uint8_t); TRANS_FUNC_DEF(vpunpckhxx); TRANS_FUNC_DEF(vpunpckhxx_lsx); TRANS_FUNC_DEF(vpunpcklxx); diff --git a/target/i386/latx/optimization/flag-reduction.c b/target/i386/latx/optimization/flag-reduction.c index c8ea963909..516004bc77 100644 --- a/target/i386/latx/optimization/flag-reduction.c +++ b/target/i386/latx/optimization/flag-reduction.c @@ -116,6 +116,9 @@ static const IR1_EFLAG_USEDEF ir1_opcode_eflag_usedef[] = { FLAG_DEFINE(VCOMISS, __NONE, __ALL_EFLAGS, __NONE), FLAG_DEFINE(VUCOMISD, __NONE, __ALL_EFLAGS, __NONE), FLAG_DEFINE(VUCOMISS, __NONE, __ALL_EFLAGS, __NONE), + FLAG_DEFINE(VPTEST, __NONE, __ALL_EFLAGS, __NONE), + FLAG_DEFINE(VTESTPS, __NONE, __ALL_EFLAGS, __NONE), + FLAG_DEFINE(VTESTPD, __NONE, __ALL_EFLAGS, __NONE), FLAG_DEFINE(IRET, __NONE, __INVALID, __NONE), /* Jcc */ diff --git a/target/i386/latx/optimization/insts-pattern.c b/target/i386/latx/optimization/insts-pattern.c index 7ebeafd1df..b20f3b209d 100644 --- a/target/i386/latx/optimization/insts-pattern.c +++ b/target/i386/latx/optimization/insts-pattern.c @@ -48,6 +48,9 @@ static inline bool ir1_can_pattern(IR1_INST *pir1) case WRAP(CDQ): case WRAP(TEST): case WRAP(UCOMISD): +#ifdef CONFIG_LATX_AVX_OPT + case WRAP(VUCOMISD): +#endif #ifdef CONFIG_LATX_SMC_OPT case WRAP(MOVAPS): case WRAP(MOVDQA): @@ -101,6 +104,9 @@ static inline bool ir1_is_pattern_head(IR1_INST *pir1) case WRAP(CDQ): case WRAP(TEST): case WRAP(UCOMISD): +#ifdef CONFIG_LATX_AVX_OPT + case WRAP(VUCOMISD): +#endif #ifdef CONFIG_LATX_SMC_OPT case WRAP(MOVAPS): case WRAP(MOVDQA): @@ -333,12 +339,23 @@ static int inst_pattern(TranslationBlock *tb, return 0; } case WRAP(UCOMISD): +#ifdef CONFIG_LATX_AVX_OPT + case WRAP(VUCOMISD): +#endif SCAN_CHECK(scan, 0); instptn_check_ucomisd_seta_0(); ir1 = SCAN_IR1(tb, scan, 0); + if (ir1_addr_next(pir1) != ir1_addr(ir1)) { + return 0; + } switch (ir1_opcode(ir1)) { case WRAP(SETA): + /* A fused store has no separate SETA fault boundary. */ + if (!ir1_opnd_is_gpr(ir1_get_opnd(ir1, 0))) { + ir1_set_eflag_def(pir1, 0x3f); + return 0; + } pir1->instptn.opc = INSTPTN_OPC_UCOMISD_SETA; pir1->instptn.next = ir1; ir1->instptn.opc = INSTPTN_OPC_NOP; @@ -443,6 +460,32 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i IR1_OPND *opnd0 = NULL; IR1_OPND *opnd1 = NULL; switch (ir1_opcode(pir1)) { +#ifdef CONFIG_LATX_AVX_OPT + case WRAP(VPTEST): + case WRAP(VTESTPS): + case WRAP(VTESTPD): + SCAN_CHECK(scan, 0); + ir1_jcc = SCAN_IR1(tb, scan, 0); + if (!option_enable_lasx || pir1_index + 1 != SCAN_IDX(scan, 0)) { + return false; + } + switch (ir1_opcode(ir1_jcc)) { + case WRAP(JE): + case WRAP(JNE): + case WRAP(JB): + case WRAP(JAE): + case WRAP(JBE): + case WRAP(JA): + instptn_check_test_jcc_0(); + pir1->instptn.opc = INSTPTN_OPC_TEST_JCC; + pir1->instptn.next = ir1_jcc; + ir1_jcc->instptn.opc = INSTPTN_OPC_NOP; + break; + default: + break; + } + return false; +#endif case WRAP(CMP): SCAN_CHECK(scan, 0); ir1_jcc = SCAN_IR1(tb, scan, 0); diff --git a/target/i386/latx/translator/mem-interface.c b/target/i386/latx/translator/mem-interface.c index a1b6731759..9566494649 100644 --- a/target/i386/latx/translator/mem-interface.c +++ b/target/i386/latx/translator/mem-interface.c @@ -367,6 +367,24 @@ static IR2_OPND convert_mem_helper(IR1_OPND *opnd1, IR2_OPND *arg_dest_op, if (host_off) { if (si12_overflow(offset)) { *host_off = 0; +#ifdef TARGET_X86_64 + bool split_offset = CODEIS64 && ir1_addr_size(pir1) == 64 && + has_base && !has_index && !has_seg && + ir1_opnd_size(opnd1) != 80; +#ifdef CONFIG_LATX_IMM_REG + split_offset = split_offset && + (!option_imm_reg || !option_imm_complex); +#endif + if (split_offset) { + longx low = sextract64(offset, 0, 12); + longx high = offset - low; + if (low && high == (int32_t)high) { + /* Keep the low displacement in the load/store itself. */ + *host_off = low; + offset = high; + } + } +#endif } else { #ifdef CONFIG_LATX_IMM_REG cache_skip = true; diff --git a/target/i386/latx/translator/tr-avx-cmp.c b/target/i386/latx/translator/tr-avx-cmp.c index 5fb062c777..c795aca14c 100644 --- a/target/i386/latx/translator/tr-avx-cmp.c +++ b/target/i386/latx/translator/tr-avx-cmp.c @@ -22,6 +22,24 @@ static inline void xcomisx(IR1_INST *pir1, bool is_double, bool qnan_exp) lsassert(ir1_opnd_num(pir1) == 2); IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); +#ifdef CONFIG_LATX_XCOMISX_OPT + uint8_t eflags = ir1_get_eflag_def(pir1) & + ~lsenv->tr_data->curr_ir1_skipped_eflags; + if (eflags != 0x3f) { + /* Keep the comparison even when its flags die: it can raise invalid. */ + if (!(eflags & (ZF_USEDEF_BIT | PF_USEDEF_BIT | CF_USEDEF_BIT))) { + if (is_double) { + la_fcmp_cond_d(fcc0_ir2_opnd, dest, src, + FCMP_COND_CUEQ + qnan_exp); + } else { + la_fcmp_cond_s(fcc0_ir2_opnd, dest, src, + FCMP_COND_CUEQ + qnan_exp); + } + } + generate_xcomisx(dest, src, is_double, qnan_exp, eflags); + return; + } +#endif /* 0. set flag = 0 */ IR2_OPND flag_zf = ra_alloc_itemp(); IR2_OPND flag_pf = ra_alloc_itemp(); @@ -56,7 +74,6 @@ static inline void xcomisx(IR1_INST *pir1, bool is_double, bool qnan_exp) ra_free_temp(flag_pf); ra_free_temp(flag_zf); ra_free_temp(flag); - } bool translate_vcomisd(IR1_INST * pir1) { @@ -198,9 +215,7 @@ bool translate_vcmpeqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_EQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_EQ); set_high128_xreg_to_zero(dest); } return true; @@ -224,9 +239,7 @@ bool translate_vcmpltpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_LT); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_LT); set_high128_xreg_to_zero(dest); } return true; @@ -250,9 +263,7 @@ bool translate_vcmplepd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_LE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_LE); set_high128_xreg_to_zero(dest); } return true; @@ -276,9 +287,7 @@ bool translate_vcmpunordpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_UNORD); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_UNORD); set_high128_xreg_to_zero(dest); } return true; @@ -302,9 +311,7 @@ bool translate_vcmpneqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NEQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_NEQ); set_high128_xreg_to_zero(dest); } return true; @@ -329,9 +336,7 @@ bool translate_vcmpnltpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_NLT); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src2, src1, X86_FCMP_COND_NLT); set_high128_xreg_to_zero(dest); } return true; @@ -355,9 +360,7 @@ bool translate_vcmpnlepd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_NLE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src2, src1, X86_FCMP_COND_NLE); set_high128_xreg_to_zero(dest); } return true; @@ -381,9 +384,7 @@ bool translate_vcmpordpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_ORD); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_ORD); set_high128_xreg_to_zero(dest); } return true; @@ -407,9 +408,7 @@ bool translate_vcmpeq_uqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_EQ_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_EQ_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -433,9 +432,7 @@ bool translate_vcmpngepd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NGE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_NGE); set_high128_xreg_to_zero(dest); } return true; @@ -459,9 +456,7 @@ bool translate_vcmpngtpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NGT); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_NGT); set_high128_xreg_to_zero(dest); } return true; @@ -485,9 +480,7 @@ bool translate_vcmpfalsepd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_FALSE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_FALSE); set_high128_xreg_to_zero(dest); } return true; @@ -511,9 +504,7 @@ bool translate_vcmpneq_oqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NEQ_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_NEQ_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -537,9 +528,7 @@ bool translate_vcmpgepd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_GE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src2, src1, X86_FCMP_COND_GE); set_high128_xreg_to_zero(dest); } return true; @@ -563,9 +552,7 @@ bool translate_vcmpgtpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_GT); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src2, src1, X86_FCMP_COND_GT); set_high128_xreg_to_zero(dest); } return true; @@ -591,10 +578,8 @@ bool translate_vcmptruepd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_TRUE); - la_xvori_b(temp, temp, 0xff); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_TRUE); + la_xvori_b(dest, dest, 0xff); set_high128_xreg_to_zero(dest); } return true; @@ -618,9 +603,7 @@ bool translate_vcmpeq_ospd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_EQ_OS); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_EQ_OS); set_high128_xreg_to_zero(dest); } return true; @@ -644,9 +627,7 @@ bool translate_vcmplt_oqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_LT_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_LT_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -670,9 +651,7 @@ bool translate_vcmple_oqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_LE_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_LE_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -696,9 +675,7 @@ bool translate_vcmpunord_spd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_UNORD_S); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_UNORD_S); set_high128_xreg_to_zero(dest); } return true; @@ -722,9 +699,7 @@ bool translate_vcmpneq_uspd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NEQ_US); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_NEQ_US); set_high128_xreg_to_zero(dest); } return true; @@ -748,9 +723,7 @@ bool translate_vcmpnlt_uqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_NLT_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src2, src1, X86_FCMP_COND_NLT_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -774,9 +747,7 @@ bool translate_vcmpnle_uqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_NLE_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src2, src1, X86_FCMP_COND_NLE_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -800,9 +771,7 @@ bool translate_vcmpord_spd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_ORD_S); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_ORD_S); set_high128_xreg_to_zero(dest); } return true; @@ -826,9 +795,7 @@ bool translate_vcmpeq_uspd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_EQ_US); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_EQ_US); set_high128_xreg_to_zero(dest); } return true; @@ -852,9 +819,7 @@ bool translate_vcmpnge_uqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NGE_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_NGE_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -878,9 +843,7 @@ bool translate_vcmpngt_uqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NGT_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_NGT_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -904,9 +867,7 @@ bool translate_vcmpfalse_ospd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_FALSE_OS); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_FALSE_OS); set_high128_xreg_to_zero(dest); } return true; @@ -930,9 +891,7 @@ bool translate_vcmpneq_ospd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NEQ_OS); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_NEQ_OS); set_high128_xreg_to_zero(dest); } return true; @@ -956,9 +915,7 @@ bool translate_vcmpge_oqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_GE_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src2, src1, X86_FCMP_COND_GE_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -982,9 +939,7 @@ bool translate_vcmpgt_oqpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_GT_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src2, src1, X86_FCMP_COND_GT_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -1010,10 +965,8 @@ bool translate_vcmptrue_uspd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_TRUE_US); - la_xvori_b(temp, temp, 0xff); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_d(dest, src1, src2, X86_FCMP_COND_TRUE_US); + la_xvori_b(dest, dest, 0xff); set_high128_xreg_to_zero(dest); } return true; @@ -1168,9 +1121,7 @@ bool translate_vcmpleps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_LE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_LE); set_high128_xreg_to_zero(dest); } return true; @@ -1194,9 +1145,7 @@ bool translate_vcmpunordps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_UNORD); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_UNORD); set_high128_xreg_to_zero(dest); } return true; @@ -1220,9 +1169,7 @@ bool translate_vcmpneqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NEQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_NEQ); set_high128_xreg_to_zero(dest); } return true; @@ -1247,9 +1194,7 @@ bool translate_vcmpnltps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_NLT); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src2, src1, X86_FCMP_COND_NLT); set_high128_xreg_to_zero(dest); } return true; @@ -1273,9 +1218,7 @@ bool translate_vcmpnleps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_NLE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src2, src1, X86_FCMP_COND_NLE); set_high128_xreg_to_zero(dest); } return true; @@ -1299,9 +1242,7 @@ bool translate_vcmpordps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_ORD); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_ORD); set_high128_xreg_to_zero(dest); } return true; @@ -1325,9 +1266,7 @@ bool translate_vcmpeq_uqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_EQ_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_EQ_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -1351,9 +1290,7 @@ bool translate_vcmpngeps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NGE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_NGE); set_high128_xreg_to_zero(dest); } return true; @@ -1377,9 +1314,7 @@ bool translate_vcmpngtps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NGT); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_NGT); set_high128_xreg_to_zero(dest); } return true; @@ -1403,9 +1338,7 @@ bool translate_vcmpfalseps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_FALSE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_FALSE); set_high128_xreg_to_zero(dest); } return true; @@ -1429,9 +1362,7 @@ bool translate_vcmpneq_oqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NEQ_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_NEQ_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -1455,9 +1386,7 @@ bool translate_vcmpgeps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_GE); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src2, src1, X86_FCMP_COND_GE); set_high128_xreg_to_zero(dest); } return true; @@ -1481,9 +1410,7 @@ bool translate_vcmpgtps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_GT); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src2, src1, X86_FCMP_COND_GT); set_high128_xreg_to_zero(dest); } return true; @@ -1509,10 +1436,8 @@ bool translate_vcmptrueps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_TRUE); - la_xvori_b(temp, temp, 0xff); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_TRUE); + la_xvori_b(dest, dest, 0xff); set_high128_xreg_to_zero(dest); } return true; @@ -1536,9 +1461,7 @@ bool translate_vcmpeq_osps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_EQ_OS); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_EQ_OS); set_high128_xreg_to_zero(dest); } return true; @@ -1562,9 +1485,7 @@ bool translate_vcmplt_oqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_LT_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_LT_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -1588,9 +1509,7 @@ bool translate_vcmple_oqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_LE_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_LE_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -1614,9 +1533,7 @@ bool translate_vcmpunord_sps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_UNORD_S); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_UNORD_S); set_high128_xreg_to_zero(dest); } return true; @@ -1640,9 +1557,7 @@ bool translate_vcmpneq_usps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NEQ_US); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_NEQ_US); set_high128_xreg_to_zero(dest); } return true; @@ -1666,9 +1581,7 @@ bool translate_vcmpnlt_uqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_NLT_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src2, src1, X86_FCMP_COND_NLT_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -1692,9 +1605,7 @@ bool translate_vcmpnle_uqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_NLE_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src2, src1, X86_FCMP_COND_NLE_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -1718,9 +1629,7 @@ bool translate_vcmpord_sps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_ORD_S); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_ORD_S); set_high128_xreg_to_zero(dest); } return true; @@ -1744,9 +1653,7 @@ bool translate_vcmpeq_usps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_EQ_US); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_EQ_US); set_high128_xreg_to_zero(dest); } return true; @@ -1770,9 +1677,7 @@ bool translate_vcmpnge_uqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NGE_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_NGE_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -1796,9 +1701,7 @@ bool translate_vcmpngt_uqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NGT_UQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_NGT_UQ); set_high128_xreg_to_zero(dest); } return true; @@ -1822,9 +1725,7 @@ bool translate_vcmpfalse_osps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_FALSE_OS); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_FALSE_OS); set_high128_xreg_to_zero(dest); } return true; @@ -1848,9 +1749,7 @@ bool translate_vcmpneq_osps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NEQ_OS); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_NEQ_OS); set_high128_xreg_to_zero(dest); } return true; @@ -1874,9 +1773,7 @@ bool translate_vcmpge_oqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_GE_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src2, src1, X86_FCMP_COND_GE_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -1900,9 +1797,7 @@ bool translate_vcmpgt_oqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_GT_OQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src2, src1, X86_FCMP_COND_GT_OQ); set_high128_xreg_to_zero(dest); } return true; @@ -1928,10 +1823,8 @@ bool translate_vcmptrue_usps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_TRUE_US); - la_xvori_b(temp, temp, 0xff); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_TRUE_US); + la_xvori_b(dest, dest, 0xff); set_high128_xreg_to_zero(dest); } return true; @@ -2020,6 +1913,7 @@ bool translate_vcmpps(IR1_INST * pir1) { return true; } +/* Scalar double compares read src1's high lane after writing the mask. */ bool translate_vcmpeqsd(IR1_INST * pir1) { if (!option_enable_lasx) { return translate_vcmpsd_lsx(pir1); @@ -2032,17 +1926,20 @@ bool translate_vcmpeqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_EQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_EQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2058,17 +1955,20 @@ bool translate_vcmpltsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_LT); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_LT); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2084,17 +1984,20 @@ bool translate_vcmplesd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_LE); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_LE); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2110,17 +2013,20 @@ bool translate_vcmpunordsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_UNORD); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_UNORD); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2136,17 +2042,20 @@ bool translate_vcmpneqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NEQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_NEQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2162,17 +2071,20 @@ bool translate_vcmpnltsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_NLT); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp2, temp1, X86_FCMP_COND_NLT); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2188,17 +2100,20 @@ bool translate_vcmpnlesd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_NLE); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp2, temp1, X86_FCMP_COND_NLE); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2214,17 +2129,20 @@ bool translate_vcmpordsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_ORD); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_ORD); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2240,17 +2158,20 @@ bool translate_vcmpeq_uqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_EQ_UQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_EQ_UQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2266,17 +2187,20 @@ bool translate_vcmpngesd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NGE); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_NGE); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2292,17 +2216,20 @@ bool translate_vcmpngtsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NGT); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_NGT); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2318,17 +2245,20 @@ bool translate_vcmpfalsesd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_FALSE); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_FALSE); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2344,17 +2274,20 @@ bool translate_vcmpneq_oqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NEQ_OQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_NEQ_OQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2370,17 +2303,20 @@ bool translate_vcmpgesd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_GE); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp2, temp1, X86_FCMP_COND_GE); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2396,17 +2332,20 @@ bool translate_vcmpgtsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_GT); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp2, temp1, X86_FCMP_COND_GT); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2422,18 +2361,21 @@ bool translate_vcmptruesd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_TRUE); - la_xvori_b(dest_temp, dest_temp, 0xff); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_TRUE); + la_xvori_b(result, result, 0xff); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2449,17 +2391,20 @@ bool translate_vcmpeq_ossd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_EQ_OS); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_EQ_OS); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2475,17 +2420,20 @@ bool translate_vcmplt_oqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_LT_OQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_LT_OQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2501,17 +2449,20 @@ bool translate_vcmple_oqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_LE_OQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_LE_OQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2527,17 +2478,20 @@ bool translate_vcmpunord_ssd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_UNORD_S); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_UNORD_S); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2553,17 +2507,20 @@ bool translate_vcmpneq_ussd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NEQ_US); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_NEQ_US); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2579,17 +2536,20 @@ bool translate_vcmpnlt_uqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_NLT_UQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp2, temp1, X86_FCMP_COND_NLT_UQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2605,17 +2565,20 @@ bool translate_vcmpnle_uqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_NLE_UQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp2, temp1, X86_FCMP_COND_NLE_UQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2631,17 +2594,20 @@ bool translate_vcmpord_ssd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_ORD_S); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_ORD_S); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2657,17 +2623,20 @@ bool translate_vcmpeq_ussd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_EQ_US); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_EQ_US); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2683,17 +2652,20 @@ bool translate_vcmpnge_uqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NGE_UQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_NGE_UQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2709,17 +2681,20 @@ bool translate_vcmpngt_uqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NGT_UQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_NGT_UQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2735,17 +2710,20 @@ bool translate_vcmpfalse_ossd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_FALSE_OS); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_FALSE_OS); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2761,17 +2739,20 @@ bool translate_vcmpneq_ossd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NEQ_OS); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_NEQ_OS); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2787,17 +2768,20 @@ bool translate_vcmpge_oqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_GE_OQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp2, temp1, X86_FCMP_COND_GE_OQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2813,17 +2797,20 @@ bool translate_vcmpgt_oqsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_GT_OQ); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp2, temp1, X86_FCMP_COND_GT_OQ); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } @@ -2839,18 +2826,21 @@ bool translate_vcmptrue_ussd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + bool can_write_dest = !ir2_opnd_cmp(&dest, &src1); + IR2_OPND result = can_write_dest ? dest : ra_alloc_ftemp(); la_vreplve_d(temp1, src1, zero_ir2_opnd); la_vreplve_d(temp2, src2, zero_ir2_opnd); - la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_TRUE_US); - la_xvori_b(dest_temp, dest_temp, 0xff); - la_vshuf4i_d(dest_temp, src1, 0xc); - la_xvori_b(dest, dest_temp, 0); + la_vfcmp_cond_d(result, temp1, temp2, X86_FCMP_COND_TRUE_US); + la_xvori_b(result, result, 0xff); + la_vshuf4i_d(result, src1, 0xc); + if (!can_write_dest) { + la_xvori_b(dest, result, 0); + ra_free_temp(result); + } set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); - ra_free_temp(dest_temp); return true; } diff --git a/target/i386/latx/translator/tr-avx-cvt.c b/target/i386/latx/translator/tr-avx-cvt.c index 4bdc082ae7..ce06913712 100644 --- a/target/i386/latx/translator/tr-avx-cvt.c +++ b/target/i386/latx/translator/tr-avx-cvt.c @@ -31,10 +31,8 @@ bool translate_vcvtps2pd(IR1_INST * pir1) { } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcvtl_d_s(temp, src); - la_xvori_b(dest, temp, 0); + la_vfcvtl_d_s(dest, src); set_high128_xreg_to_zero(dest); } return true; @@ -57,11 +55,13 @@ bool translate_vcvtpd2ps(IR1_INST * pir1) { } else if (ir1_opnd_size(ir1_get_opnd(pir1, 1)) == 256) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND result = ir2_opnd_cmp(&dest, &src) ? + ra_alloc_ftemp() : dest; - la_xvpermi_q(temp, src, XVPERMI_Q_4_0(1, 1)); - la_vfcvt_s_d(temp, temp, src); - la_xvori_b(dest, temp, 0); + la_xvpermi_q(result, src, XVPERMI_Q_4_0(1, 1)); + la_vfcvt_s_d(result, result, src); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); set_high128_xreg_to_zero(dest); } else { lsassert(0); @@ -86,10 +86,8 @@ bool translate_vcvtdq2ps(IR1_INST * pir1) { } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vffint_s_w(temp, src); - la_xvori_b(dest, temp, 0); + la_vffint_s_w(dest, src); set_high128_xreg_to_zero(dest); } return true; @@ -361,20 +359,17 @@ bool translate_vcvtdq2pd(IR1_INST * pir1) { if (ir1_opnd_is_ymm(ir1_get_opnd(pir1, 0))) { IR2_OPND dest = load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); + IR2_OPND high = ra_alloc_ftemp(); - la_vffintl_d_w(temp1, src); - la_vffinth_d_w(temp2, src); - la_xvpermi_q(temp1, temp2, XVPERMI_Q_4_0(0, 2)); - la_xvori_b(dest, temp1, 0); + /* Read the overlapping source before overwriting dest's low half. */ + la_vffinth_d_w(high, src); + la_vffintl_d_w(dest, src); + la_xvpermi_q(dest, high, XVPERMI_Q_4_0(0, 2)); } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vffintl_d_w(temp, src); - la_xvori_b(dest, temp, 0); + la_vffintl_d_w(dest, src); set_high128_xreg_to_zero(dest); } set_fpu_rounding_mode(fcsr_opnd); @@ -387,7 +382,9 @@ static bool translate_vcvtpd2dq_opt(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src = load_freg256_from_ir1(opnd1); - IR2_OPND temp_f = ra_alloc_ftemp(); + bool can_write_dest = ir1_opnd_size(opnd1) == 256 && + !ir2_opnd_cmp(&dest, &src); + IR2_OPND temp_f = can_write_dest ? dest : ra_alloc_ftemp(); IR2_OPND mxcsr = ra_alloc_itemp(); IR2_OPND temp_i = ra_alloc_itemp(); int offset = lsenv_offset_of_mxcsr(lsenv); @@ -435,7 +432,8 @@ static bool translate_vcvtpd2dq_opt(IR1_INST * pir1) { la_xvpermi_d(comp_mask, comp_mask, 0x88); // la_xvshuf4i_d(comp_mask, comp_mask, 0x88); la_vbitsel_v(temp_f, temp_f, sse_invalid, comp_mask); - la_vand_v(dest, temp_f, temp_f); + if (!ir2_opnd_cmp(&temp_f, &dest)) + la_vand_v(dest, temp_f, temp_f); set_high128_xreg_to_zero(dest); } return true; @@ -584,11 +582,11 @@ bool translate_vcvtsd2ss(IR1_INST * pir1) { IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND dest_temp = ra_alloc_ftemp(); + la_fcvt_s_d(temp, src2); - la_vori_b(dest_temp, src1, 0); - la_xvinsve0_w(dest_temp, temp, 0); - la_xvori_b(dest, dest_temp, 0); + if (!ir2_opnd_cmp(&dest, &src1)) + la_vori_b(dest, src1, 0); + la_xvinsve0_w(dest, temp, 0); set_high128_xreg_to_zero(dest); set_fpu_rounding_mode(fcsr_opnd); return true; @@ -607,17 +605,22 @@ bool translate_vcvtsi2sd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_ireg_from_ir1(opnd2, UNKNOWN_EXTENSION, false); - IR2_OPND temp = ra_alloc_ftemp(); - la_movgr2fr_d(temp, src2); + IR2_OPND result = ir2_opnd_cmp(&dest, &src1) ? + ra_alloc_ftemp() : dest; + + la_movgr2fr_d(result, src2); if (ir1_opnd_size(opnd2) == 32) { - la_vffintl_d_w(temp, temp); + la_vffintl_d_w(result, result); } else if (ir1_opnd_size(opnd2) == 64) { - la_vffint_d_l(temp, temp); + la_vffint_d_l(result, result); } else { lsassert(0); } - la_vshuf4i_d(temp, src1, 0xc); - la_xvori_b(dest, temp, 0); + if (ir2_opnd_cmp(&dest, &src1)) { + la_xvinsve0_d(dest, result, 0); + } else { + la_vshuf4i_d(dest, src1, 0xc); + } set_high128_xreg_to_zero(dest); return true; } @@ -632,10 +635,15 @@ bool translate_vcvtss2sd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_fcvt_d_s(temp, src2); - la_vshuf4i_d(temp, src1, 0xc); - la_xvori_b(dest, temp, 0); + IR2_OPND result = ir2_opnd_cmp(&dest, &src1) ? + ra_alloc_ftemp() : dest; + + la_fcvt_d_s(result, src2); + if (ir2_opnd_cmp(&dest, &src1)) { + la_xvinsve0_d(dest, result, 0); + } else { + la_vshuf4i_d(dest, src1, 0xc); + } set_high128_xreg_to_zero(dest); return true; } @@ -675,7 +683,9 @@ static bool translate_vcvttpd2dq_opt(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src = load_freg256_from_ir1(opnd1); - IR2_OPND temp_f = ra_alloc_ftemp(); + bool can_write_dest = ir1_opnd_size(opnd1) == 256 && + !ir2_opnd_cmp(&dest, &src); + IR2_OPND temp_f = can_write_dest ? dest : ra_alloc_ftemp(); IR2_OPND temp_i = ra_alloc_itemp(); IR2_OPND sse_invalid = ra_alloc_ftemp(); IR2_OPND overflow = ra_alloc_ftemp(); @@ -701,7 +711,8 @@ static bool translate_vcvttpd2dq_opt(IR1_INST * pir1) { la_xvshuf4i_w(comp_mask, comp_mask, 0x88); la_xvpermi_d(comp_mask, comp_mask, 0x88); la_vbitsel_v(temp_f, temp_f, sse_invalid, comp_mask); - la_vand_v(dest, temp_f, temp_f); + if (!ir2_opnd_cmp(&temp_f, &dest)) + la_vand_v(dest, temp_f, temp_f); set_high128_xreg_to_zero(dest); } return true; diff --git a/target/i386/latx/translator/tr-avx-mov.c b/target/i386/latx/translator/tr-avx-mov.c index baaa387923..815a1ee950 100644 --- a/target/i386/latx/translator/tr-avx-mov.c +++ b/target/i386/latx/translator/tr-avx-mov.c @@ -75,8 +75,9 @@ bool translate_vmovaps(IR1_INST * pir1) { store_freg256_to_ir1_mem(ra_alloc_xmm(ir1_opnd_base_reg_num(src)), dest); } else if (ir1_opnd_is_ymm(dest) && ir1_opnd_is_ymm(src)) { - la_xvori_b(ra_alloc_xmm(ir1_opnd_base_reg_num(dest)), - ra_alloc_xmm(ir1_opnd_base_reg_num(src)), 0); + if (!ir1_opnd_is_same_reg(dest, src)) + la_xvori_b(ra_alloc_xmm(ir1_opnd_base_reg_num(dest)), + ra_alloc_xmm(ir1_opnd_base_reg_num(src)), 0); } else if (ir1_opnd_is_xmm(dest) && ir1_opnd_is_mem(src)) { IR2_OPND dest_reg = ra_alloc_xmm(ir1_opnd_base_reg_num(dest)); @@ -88,7 +89,8 @@ bool translate_vmovaps(IR1_INST * pir1) { } else if (ir1_opnd_is_xmm(dest) && ir1_opnd_is_xmm(src)) { IR2_OPND dest_reg = ra_alloc_xmm(ir1_opnd_base_reg_num(dest)); - la_vori_b(dest_reg, ra_alloc_xmm(ir1_opnd_base_reg_num(src)), 0); + if (!ir1_opnd_is_same_reg(dest, src)) + la_vori_b(dest_reg, ra_alloc_xmm(ir1_opnd_base_reg_num(src)), 0); set_high128_xreg_to_zero(dest_reg); } else { #ifdef CONFIG_LATX_TS @@ -113,15 +115,14 @@ bool translate_vmovmskps(IR1_INST * pir1) { ra_alloc_xmm(ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1)))); la_movfr2gr_d(dest, temp); } else { - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); + IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND high = ra_alloc_itemp(); - la_xvmskltz_w(temp1, + la_xvmskltz_w(temp, ra_alloc_xmm(ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1)))); - la_xvpermi_q(temp2, temp1, VEXTRINS_IMM_4_0(1, 1)); - la_vslli_b(temp2, temp2, 4); - la_vor_v(temp1, temp1, temp2); - la_movfr2gr_d(dest, temp1); + la_xvpickve2gr_du(dest, temp, 0); + la_xvpickve2gr_du(high, temp, 2); + la_bstrins_d(dest, high, 7, 4); } return true; } @@ -140,15 +141,14 @@ bool translate_vmovmskpd(IR1_INST * pir1) { ra_alloc_xmm(ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1)))); la_movfr2gr_d(dest, temp); } else { - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); + IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND high = ra_alloc_itemp(); - la_xvmskltz_d(temp1, + la_xvmskltz_d(temp, ra_alloc_xmm(ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1)))); - la_xvpermi_q(temp2, temp1, VEXTRINS_IMM_4_0(1, 1)); - la_vslli_b(temp2, temp2, 2); - la_vor_v(temp1, temp1, temp2); - la_movfr2gr_d(dest, temp1); + la_xvpickve2gr_du(dest, temp, 0); + la_xvpickve2gr_du(high, temp, 2); + la_bstrins_d(dest, high, 3, 2); } return true; } @@ -207,6 +207,7 @@ bool translate_vmovshdup(IR1_INST * pir1) { } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); + la_vpackod_w(dest, src, src); set_high128_xreg_to_zero(dest); } @@ -227,10 +228,8 @@ bool translate_vmovsldup(IR1_INST * pir1) { } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvpackev_w(temp, src, src); - la_xvori_b(dest, temp, 0); + la_xvpackev_w(dest, src, src); set_high128_xreg_to_zero(dest); } return true; @@ -251,10 +250,8 @@ bool translate_vmovddup(IR1_INST * pir1) { } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvrepl128vei_d(temp, src, 0); - la_xvori_b(dest, temp, 0); + la_xvrepl128vei_d(dest, src, 0); set_high128_xreg_to_zero(dest); } return true; @@ -310,16 +307,20 @@ bool translate_vmovss(IR1_INST *pir1) IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); if (ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_xmm(opnd1)) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - la_xvori_b(temp, src1, 0); - la_xvinsve0_w(temp, src2, 0); - la_xvori_b(dest, temp, 0); + IR2_OPND result = ir2_opnd_cmp(&dest, &src2) ? + ra_alloc_ftemp() : dest; + + if (!ir2_opnd_cmp(&result, &src1)) + la_vori_b(result, src1, 0); + la_xvinsve0_w(result, src2, 0); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); set_high128_xreg_to_zero(dest); } else if (ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_mem(opnd1)) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); la_xvpickve_w(dest, src1, 0); - set_high128_xreg_to_zero(dest); + cancel_pending_ymmh_zero(dest); } else if (ir1_opnd_is_mem(opnd0) && ir1_opnd_is_xmm(opnd1)) { int little_disp; IR2_OPND mem_opnd = convert_mem(opnd0, &little_disp); @@ -346,9 +347,8 @@ bool translate_vmovd(IR1_INST *pir1) if (ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_mem(opnd1)) { IR2_OPND src = load_freg_from_ir1_1(opnd1, false, IS_INTEGER); IR2_OPND dest = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd0)); - la_xvandi_b(dest, dest, 0); la_xvpickve_w(dest, src, 0); - set_high128_xreg_to_zero(dest); + cancel_pending_ymmh_zero(dest); } else if (ir1_opnd_is_mem(opnd0) && ir1_opnd_is_xmm(opnd1)) { IR2_OPND src = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd1)); store_freg_to_ir1(src, opnd0, false, false); @@ -357,7 +357,7 @@ bool translate_vmovd(IR1_INST *pir1) IR2_OPND src = load_ireg_from_ir1(opnd1, UNKNOWN_EXTENSION, false); la_xvandi_b(dest, dest, 0); la_xvinsgr2vr_w(dest, src, 0); - set_high128_xreg_to_zero(dest); + cancel_pending_ymmh_zero(dest); } else if (ir1_opnd_is_gpr(opnd0) && ir1_opnd_is_xmm(opnd1)) { IR2_OPND dest = load_ireg_from_ir1(opnd0, UNKNOWN_EXTENSION, false); IR2_OPND src = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd1)); @@ -536,8 +536,12 @@ bool translate_vmovlpd(IR1_INST * pir1) { IR2_OPND temp_src2 = load_freg128_from_ir1(src2); IR2_OPND dest_temp = ra_alloc_xmm(ir1_opnd_base_reg_num(dest)); IR2_OPND src_temp = ra_alloc_xmm(ir1_opnd_base_reg_num(src)); - la_vreplvei_d(dest_temp, src_temp, 1); - la_xvinsve0_d(dest_temp, temp_src2, 0); + if (ir1_opnd_is_same_reg(dest, src)) { + la_vshuf4i_d(dest_temp, temp_src2, 0x6); + } else { + la_vreplvei_d(dest_temp, src_temp, 1); + la_xvinsve0_d(dest_temp, temp_src2, 0); + } set_high128_xreg_to_zero(dest_temp); return true; } else if (ir1_opnd_is_mem(dest) && ir1_opnd_is_xmm(src)) { @@ -615,20 +619,7 @@ bool translate_vmovntdqa(IR1_INST * pir1) { if (!option_enable_lasx) { return translate_vmovntdqa_lsx(pir1); } - IR1_OPND * dest = ir1_get_opnd(pir1, 0); - IR1_OPND * src = ir1_get_opnd(pir1, 1); - if (ir1_opnd_is_xmm(dest) && ir1_opnd_is_mem(src)) { - IR2_OPND temp = load_freg128_from_ir1(src); - IR2_OPND temp_dest = ra_alloc_xmm(ir1_opnd_base_reg_num(dest)); - la_vand_v(temp_dest, temp, temp); - set_high128_xreg_to_zero(temp_dest); - - } else if (ir1_opnd_is_ymm(dest) && ir1_opnd_is_mem(src)) { - IR2_OPND temp = load_freg256_from_ir1(src); - la_xvand_v(ra_alloc_xmm(ir1_opnd_base_reg_num(dest)), - temp, temp); - } - return true; + return translate_vmovaps(pir1); } bool translate_vmovsd(IR1_INST * pir1) { @@ -644,11 +635,19 @@ bool translate_vmovsd(IR1_INST * pir1) { IR2_OPND temp_src1 = load_freg128_from_ir1(src); IR2_OPND temp_src2 = load_freg128_from_ir1(src2); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvori_b(temp, temp_src1, 0x0); - la_vshuf4i_d(temp, temp_src2, 0b00000110); - la_xvori_b(temp_dest, temp, 0x0); + if (ir2_opnd_cmp(&temp_src1, &temp_src2)) { + if (!ir2_opnd_cmp(&temp_dest, &temp_src1)) { + la_vori_b(temp_dest, temp_src1, 0); + } + } else if (ir2_opnd_cmp(&temp_dest, &temp_src2)) { + la_vextrins_d(temp_dest, temp_src1, VEXTRINS_IMM_4_0(1, 1)); + } else { + if (!ir2_opnd_cmp(&temp_dest, &temp_src1)) { + la_vori_b(temp_dest, temp_src1, 0); + } + la_vshuf4i_d(temp_dest, temp_src2, 0b00000110); + } set_high128_xreg_to_zero(temp_dest); return true; } diff --git a/target/i386/latx/translator/tr-avx-shift.c b/target/i386/latx/translator/tr-avx-shift.c index fc666fac30..b185cc6d37 100644 --- a/target/i386/latx/translator/tr-avx-shift.c +++ b/target/i386/latx/translator/tr-avx-shift.c @@ -150,8 +150,6 @@ bool translate_vpsrlx(IR1_INST * pir1) { IR2_INST * ( * rep_inst)(IR2_OPND, IR2_OPND); IR2_INST * ( * tr_inst_i)(IR2_OPND, IR2_OPND, int); IR2_INST * ( * tr_inst_r)(IR2_OPND, IR2_OPND, IR2_OPND); - IR2_OPND label_exit = ra_alloc_label(); - IR2_OPND label_shift = ra_alloc_label(); int max_count; switch (ir1_opcode(pir1)) { case dt_X86_INS_VPSRLW: @@ -192,18 +190,6 @@ bool translate_vpsrlx(IR1_INST * pir1) { IR2_OPND mask = ra_alloc_ftemp(); IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND count = ra_alloc_itemp(); - IR2_OPND max = ra_alloc_itemp(); - /* The low 64 bits of the count operand are one scalar shift count; - * a count equal to max_count still shifts, only a larger count - * yields zero. Compare against max_count + 1 to keep "==" valid. */ - la_addi_d(max, zero_ir2_opnd, max_count + 1); - la_vpickve2gr_d(count, src2, 0); - la_blt(count, max, label_shift); - la_xvxor_v(dest, dest, dest); - la_b(label_exit); - - la_label(label_shift); if (max_count == 63) { la_xvreplve0_d(mask, src2); la_xvldi(temp, VLDI_IMM_TYPE0(3, 63)); @@ -219,7 +205,6 @@ bool translate_vpsrlx(IR1_INST * pir1) { if (ir1_opnd_is_xmm(opnd0)) { set_high128_xreg_to_zero(dest); } - la_label(label_exit); return true; } @@ -267,21 +252,11 @@ bool translate_vpsrax(IR1_INST * pir1) { tr_inst_i(dest, src, imm); } else { IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND mask = ra_alloc_ftemp(); IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND temp_sign = ra_alloc_ftemp(); - if (max_count == 63) { - la_xvreplve0_d(mask, src2); - la_vldi(temp, VLDI_IMM_TYPE0(3, 63)); - la_xvsle_du(mask, mask, temp); - } else { - la_xvreplve0_d(mask, src2); - la_xvslei_du(mask, mask, max_count); - } - tr_inst_i(temp_sign, src, max_count); - rep_inst(temp, src2); - tr_inst_r(temp, src, temp); - la_xvbitsel_v(dest, temp_sign, temp, mask); + + la_xvmini_du(temp, src2, max_count); + rep_inst(temp, temp); + tr_inst_r(dest, src, temp); } if (ir1_opnd_is_xmm(opnd0)) { set_high128_xreg_to_zero(dest); diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index c2109b96da..dcb41e143f 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -34,13 +34,11 @@ bool translate_vaddpd(IR1_INST * pir1) { src2 = load_freg256_from_ir1(opnd2); la_xvfadd_d(dest, src1, src2); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); - la_vfadd_d(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vfadd_d(dest, src1, src2); set_high128_xreg_to_zero(dest); } + ra_free_temp_auto(src2); return true; } @@ -69,6 +67,7 @@ bool translate_vaddps(IR1_INST * pir1) { la_vfadd_s(dest, src1, src2); set_high128_xreg_to_zero(dest); } + ra_free_temp_auto(src2); return true; } @@ -83,11 +82,15 @@ bool translate_vaddsd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND result = ir2_opnd_cmp(&dest, &src1) ? + ra_alloc_ftemp() : dest; - la_fadd_d(temp, src1, src2); - la_vshuf4i_d(temp, src1, 0xc); - la_xvori_b(dest, temp, 0); + la_fadd_d(result, src1, src2); + if (ir2_opnd_cmp(&dest, &src1)) { + la_xvinsve0_d(dest, result, 0); + } else { + la_vshuf4i_d(dest, src1, 0xc); + } set_high128_xreg_to_zero(dest); return true; } @@ -136,13 +139,11 @@ bool translate_vsubpd(IR1_INST * pir1) { src2 = load_freg256_from_ir1(opnd2); la_xvfsub_d(dest, src1, src2); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); - la_vfsub_d(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vfsub_d(dest, src1, src2); set_high128_xreg_to_zero(dest); } + ra_free_temp_auto(src2); return true; } @@ -171,6 +172,7 @@ bool translate_vsubps(IR1_INST * pir1) { la_vfsub_s(dest, src1, src2); set_high128_xreg_to_zero(dest); } + ra_free_temp_auto(src2); return true; } @@ -187,10 +189,15 @@ bool translate_vsubsd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); - la_fsub_d(temp, src1, src2); - la_vshuf4i_d(temp, src1, 0xc); - la_xvori_b(dest, temp, 0); + IR2_OPND result = ir2_opnd_cmp(&dest, &src1) ? + ra_alloc_ftemp() : dest; + + la_fsub_d(result, src1, src2); + if (ir2_opnd_cmp(&dest, &src1)) { + la_xvinsve0_d(dest, result, 0); + } else { + la_vshuf4i_d(dest, src1, 0xc); + } set_high128_xreg_to_zero(dest); return true; } @@ -239,13 +246,11 @@ bool translate_vmulpd(IR1_INST * pir1) { src2 = load_freg256_from_ir1(opnd2); la_xvfmul_d(dest, src1, src2); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); - la_vfmul_d(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vfmul_d(dest, src1, src2); set_high128_xreg_to_zero(dest); } + ra_free_temp_auto(src2); return true; } @@ -274,6 +279,7 @@ bool translate_vmulps(IR1_INST * pir1) { la_vfmul_s(dest, src1, src2); set_high128_xreg_to_zero(dest); } + ra_free_temp_auto(src2); return true; } @@ -287,11 +293,15 @@ bool translate_vmulsd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND result = ir2_opnd_cmp(&dest, &src1) ? + ra_alloc_ftemp() : dest; - la_fmul_d(temp, src1, src2); - la_vshuf4i_d(temp, src1, 0xc); - la_xvori_b(dest, temp, 0); + la_fmul_d(result, src1, src2); + if (ir2_opnd_cmp(&dest, &src1)) { + la_xvinsve0_d(dest, result, 0); + } else { + la_vshuf4i_d(dest, src1, 0xc); + } set_high128_xreg_to_zero(dest); return true; } @@ -340,13 +350,11 @@ bool translate_vdivpd(IR1_INST * pir1) { src2 = load_freg256_from_ir1(opnd2); la_xvfdiv_d(dest, src1, src2); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); - la_vfdiv_d(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vfdiv_d(dest, src1, src2); set_high128_xreg_to_zero(dest); } + ra_free_temp_auto(src2); return true; } @@ -375,6 +383,7 @@ bool translate_vdivps(IR1_INST * pir1) { la_vfdiv_s(dest, src1, src2); set_high128_xreg_to_zero(dest); } + ra_free_temp_auto(src2); return true; } @@ -388,10 +397,15 @@ bool translate_vdivsd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_fdiv_d(temp, src1, src2); - la_vshuf4i_d(temp, src1, 0xc); - la_xvori_b(dest, temp, 0); + IR2_OPND result = ir2_opnd_cmp(&dest, &src1) ? + ra_alloc_ftemp() : dest; + + la_fdiv_d(result, src1, src2); + if (ir2_opnd_cmp(&dest, &src1)) { + la_xvinsve0_d(dest, result, 0); + } else { + la_vshuf4i_d(dest, src1, 0xc); + } set_high128_xreg_to_zero(dest); return true; } @@ -436,10 +450,8 @@ bool translate_vsqrtpd(IR1_INST * pir1) { } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfsqrt_d(temp, src); - la_xvori_b(dest, temp, 0); + la_vfsqrt_d(dest, src); set_high128_xreg_to_zero(dest); } return true; @@ -462,10 +474,8 @@ bool translate_vsqrtps(IR1_INST * pir1) { } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfsqrt_s(temp, src); - la_xvori_b(dest, temp, 0); + la_vfsqrt_s(dest, src); set_high128_xreg_to_zero(dest); } return true; @@ -531,6 +541,19 @@ bool translate_vaddsubpd(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(opnd2); + if (ir1_opnd_is_xmm(opnd0)) { + IR2_OPND low = ra_alloc_ftemp(); + IR2_OPND high1 = ra_alloc_ftemp(); + IR2_OPND high2 = ra_alloc_ftemp(); + + la_fsub_d(low, src1, src2); + la_xvpickve_d(high1, src1, 1); + la_xvpickve_d(high2, src2, 1); + la_fadd_d(high1, high1, high2); + la_vilvl_d(dest, high1, low); + set_high128_xreg_to_zero(dest); + return true; + } IR2_OPND add_src1 = ra_alloc_ftemp(); IR2_OPND add_src2 = ra_alloc_ftemp(); IR2_OPND sub_src1 = ra_alloc_ftemp(); @@ -540,17 +563,9 @@ bool translate_vaddsubpd(IR1_INST * pir1) { la_xvpackev_d(sub_src2, src2, src2); la_xvpackod_d(add_src1, src1, src1); la_xvpackod_d(add_src2, src2, src2); - if (ir1_opnd_is_xmm(opnd0)) { - la_vfsub_d(sub_src1, sub_src1, sub_src2); - la_vfadd_d(add_src1, add_src1, add_src2); - } else { - la_xvfsub_d(sub_src1, sub_src1, sub_src2); - la_xvfadd_d(add_src1, add_src1, add_src2); - } + la_xvfsub_d(sub_src1, sub_src1, sub_src2); + la_xvfadd_d(add_src1, add_src1, add_src2); la_xvpackev_d(dest, add_src1, sub_src1); - if (ir1_opnd_is_xmm(opnd0)) { - set_high128_xreg_to_zero(dest); - } return true; } @@ -618,14 +633,12 @@ bool translate_vhaddpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); la_vpickev_d(temp1, src2, src1); la_vpickod_d(temp2, src2, src1); - la_vfadd_d(temp, temp1, temp2); - la_xvori_b(dest, temp, 0); + la_vfadd_d(dest, temp1, temp2); set_high128_xreg_to_zero(dest); } return true; @@ -657,14 +670,12 @@ bool translate_vhaddps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); la_vpickev_w(temp1, src2, src1); la_vpickod_w(temp2, src2, src1); - la_vfadd_s(temp, temp1, temp2); - la_xvori_b(dest, temp, 0); + la_vfadd_s(dest, temp1, temp2); set_high128_xreg_to_zero(dest); } return true; @@ -696,14 +707,12 @@ bool translate_vhsubpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); la_vpickev_d(temp1, src2, src1); la_vpickod_d(temp2, src2, src1); - la_vfsub_d(temp, temp1, temp2); - la_xvori_b(dest, temp, 0); + la_vfsub_d(dest, temp1, temp2); set_high128_xreg_to_zero(dest); } return true; @@ -735,14 +744,12 @@ bool translate_vhsubps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); la_vpickev_w(temp1, src2, src1); la_vpickod_w(temp2, src2, src1); - la_vfsub_s(temp, temp1, temp2); - la_xvori_b(dest, temp, 0); + la_vfsub_s(dest, temp1, temp2); set_high128_xreg_to_zero(dest); } return true; @@ -768,10 +775,8 @@ bool translate_vandnpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vandn_v(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vandn_v(dest, src1, src2); set_high128_xreg_to_zero(dest); } return true; @@ -806,6 +811,7 @@ bool translate_vandpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); + la_vand_v(dest, src1, src2); set_high128_xreg_to_zero(dest); } @@ -841,10 +847,8 @@ bool translate_vorps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vor_v(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vor_v(dest, src1, src2); set_high128_xreg_to_zero(dest); } return true; @@ -879,10 +883,8 @@ bool translate_vxorps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vxor_v(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vxor_v(dest, src1, src2); set_high128_xreg_to_zero(dest); } return true; @@ -1102,8 +1104,7 @@ bool translate_vblendvpd(IR1_INST * pir1) { IR2_OPND src3 = load_freg128_from_ir1(opnd3); IR2_OPND temp = ra_alloc_ftemp(); la_vslti_d(temp, src3, 0); - la_vbitsel_v(temp, src1, src2, temp); - la_xvori_b(dest, temp, 0); + la_vbitsel_v(dest, src1, src2, temp); set_high128_xreg_to_zero(dest); } return true; @@ -1124,24 +1125,75 @@ bool translate_vblendpd(IR1_INST * pir1) { IR2_OPND src1 = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); uint8 imm = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvori_b(temp, src1, 0); - if (imm & 0x1) - la_vextrins_d(temp, src2, VEXTRINS_IMM_4_0(0, 0)); - if (imm & 0x2) - la_vextrins_d(temp, src2, VEXTRINS_IMM_4_0(1, 1)); - if (ir1_opnd_is_ymm(ir1_get_opnd(pir1, 0))) { - IR2_OPND temp2 = ra_alloc_ftemp(); - la_xvori_b(temp2, src1, 0); - if (imm & 0x4) - la_xvextrins_d(temp2, src2, VEXTRINS_IMM_4_0(0, 0)); - if (imm & 0x8) - la_xvextrins_d(temp2, src2, VEXTRINS_IMM_4_0(1, 1)); - la_xvpermi_q(temp, temp2, VEXTRINS_IMM_4_0(1, 2)); + uint8 lane_mask = ir1_opnd_is_ymm(ir1_get_opnd(pir1, 0)) ? + 0xf : 0x3; + + if ((imm & lane_mask) == 0) { + if (!ir2_opnd_cmp(&dest, &src1)) + la_xvori_b(dest, src1, 0); + if (ir1_opnd_is_xmm(ir1_get_opnd(pir1, 0))) + set_high128_xreg_to_zero(dest); + return true; } - la_xvori_b(dest, temp, 0); - if (ir1_opnd_is_xmm(ir1_get_opnd(pir1, 0))) + if ((imm & lane_mask) == lane_mask) { + if (!ir2_opnd_cmp(&dest, &src2)) + la_xvori_b(dest, src2, 0); + if (ir1_opnd_is_xmm(ir1_get_opnd(pir1, 0))) + set_high128_xreg_to_zero(dest); + return true; + } + + if (ir1_opnd_is_xmm(ir1_get_opnd(pir1, 0))) { + if (ir2_opnd_cmp(&dest, &src1) && ir2_opnd_cmp(&dest, &src2)) { + set_high128_xreg_to_zero(dest); + return true; + } + if (ir2_opnd_cmp(&dest, &src2) && !ir2_opnd_cmp(&dest, &src1)) { + if (!(imm & 0x1)) + la_vextrins_d(dest, src1, VEXTRINS_IMM_4_0(0, 0)); + if (!(imm & 0x2)) + la_vextrins_d(dest, src1, VEXTRINS_IMM_4_0(1, 1)); + set_high128_xreg_to_zero(dest); + return true; + } + IR2_OPND result = ir2_opnd_cmp(&dest, &src2) ? + ra_alloc_ftemp() : dest; + + if (!ir2_opnd_cmp(&result, &src1)) + la_vori_b(result, src1, 0); + if (imm & 0x1) + la_vextrins_d(result, src2, VEXTRINS_IMM_4_0(0, 0)); + if (imm & 0x2) + la_vextrins_d(result, src2, VEXTRINS_IMM_4_0(1, 1)); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); set_high128_xreg_to_zero(dest); + return true; + } + + bool use_src2_base = __builtin_popcount(imm & lane_mask) > 2; + uint8 insert_mask = use_src2_base ? (~imm & lane_mask) : + (imm & lane_mask); + IR2_OPND base = use_src2_base ? src2 : src1; + IR2_OPND insert_src = use_src2_base ? src1 : src2; + IR2_OPND result = (!ir2_opnd_cmp(&dest, &src1) && + !ir2_opnd_cmp(&dest, &src2)) ? + dest : ra_alloc_ftemp(); + + la_xvori_b(result, base, 0); + if (insert_mask & 0x1) + la_vextrins_d(result, insert_src, VEXTRINS_IMM_4_0(0, 0)); + if (insert_mask & 0x2) + la_vextrins_d(result, insert_src, VEXTRINS_IMM_4_0(1, 1)); + IR2_OPND high = ra_alloc_ftemp(); + la_xvori_b(high, base, 0); + if (insert_mask & 0x4) + la_xvextrins_d(high, insert_src, VEXTRINS_IMM_4_0(0, 0)); + if (insert_mask & 0x8) + la_xvextrins_d(high, insert_src, VEXTRINS_IMM_4_0(1, 1)); + la_xvpermi_q(result, high, VEXTRINS_IMM_4_0(1, 2)); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); return true; } @@ -1150,48 +1202,62 @@ bool translate_vblendps(IR1_INST * pir1) { return translate_vblendps_lsx(pir1); } + IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); + IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); + IR1_OPND * opnd2 = ir1_get_opnd(pir1, 2); lsassert(ir1_opnd_num(pir1) == 4 && ir1_opnd_is_imm(ir1_get_opnd(pir1, 3))); - lsassert((ir1_opnd_is_xmm(ir1_get_opnd(pir1, 0)) && - ir1_opnd_is_xmm(ir1_get_opnd(pir1, 1))) || - (ir1_opnd_is_ymm(ir1_get_opnd(pir1, 0)) && - ir1_opnd_is_ymm(ir1_get_opnd(pir1, 1)))); - IR2_OPND dest = load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); - IR2_OPND src1 = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); + lsassert((ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_xmm(opnd1)) || + (ir1_opnd_is_ymm(opnd0) && ir1_opnd_is_ymm(opnd1))); + IR2_OPND dest = load_freg256_from_ir1(opnd0); + IR2_OPND src1 = load_freg256_from_ir1(opnd1); + IR2_OPND src2 = load_freg256_from_ir1(opnd2); uint8 imm = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)); + int lane_count = ir1_opnd_is_ymm(opnd0) ? 8 : 4; + uint8_t all_lanes_mask = (1u << lane_count) - 1; + uint8_t lane_mask = imm & all_lanes_mask; + bool dest_is_src2 = ir1_opnd_is_same_reg(opnd0, opnd2); + bool dest_is_src1 = ir1_opnd_is_same_reg(opnd0, opnd1); + uint8_t insert_mask = dest_is_src2 ? ~lane_mask : lane_mask; + int insert_count = __builtin_popcount(insert_mask & + all_lanes_mask); - if (ir1_opnd_is_xmm(ir1_get_opnd(pir1, 0)) && - ir1_opnd_is_xmm(ir1_get_opnd(pir1, 2)) && - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 2))) { - if (ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) { - la_vori_b(dest, src1, 0); + /* Both sources are loaded before these copies, including a memory src2. */ + if (lane_mask == 0) { + if (!dest_is_src1) + la_xvori_b(dest, src1, 0); + } else if (lane_mask == all_lanes_mask) { + if (!dest_is_src2) + la_xvori_b(dest, src2, 0); + } else if (ir1_opnd_is_xmm(opnd0) && + insert_count + (!dest_is_src2 && !dest_is_src1) <= 3) { + IR2_OPND insert_src = dest_is_src2 ? src1 : src2; + + if (!dest_is_src2 && !dest_is_src1) { + la_xvori_b(dest, src1, 0); } - for (int i = 0; i < 4; ++i) { - if (imm & (1 << i)) { - la_vextrins_w(dest, src2, VEXTRINS_IMM_4_0(i, i)); + for (int i = 0; i < lane_count; i++) { + if (insert_mask & (1u << i)) { + la_vextrins_w(dest, insert_src, + VEXTRINS_IMM_4_0(i, i)); } } - set_high128_xreg_to_zero(dest); - return true; - } - - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND rmask = ra_alloc_itemp(); + } else { + IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND rmask = ra_alloc_itemp(); + uint64_t mask = 0; - uint64_t mask = 0; - for (int i = 0; i < 8; i++) { - if (imm & (1 << i)) { - mask |= (0xFFULL) << (i * 8); + for (int i = 0; i < 8; i++) { + if (imm & (1 << i)) { + mask |= (0xFFULL) << (i * 8); + } } - } - li_d(rmask, mask); - la_movgr2fr_d(temp, rmask); - la_vext2xv_w_b(temp, temp); - la_xvbitsel_v(dest, src1, src2, temp); + li_d(rmask, mask); + la_movgr2fr_d(temp, rmask); + la_vext2xv_w_b(temp, temp); + la_xvbitsel_v(dest, src1, src2, temp); + } if (ir1_opnd_is_xmm(ir1_get_opnd(pir1, 0))) set_high128_xreg_to_zero(dest); @@ -1232,7 +1298,28 @@ bool translate_vbroadcastsd(IR1_INST * pir1) { lsassert(ir1_opnd_is_ymm(ir1_get_opnd(pir1, 0))); IR2_OPND dest = load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); - IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); + IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); + + if (ir1_opnd_is_mem(opnd1)) { + int disp; + IR2_OPND address = convert_mem(opnd1, &disp); + + gen_test_page_flag(address, disp, PAGE_READ, 8); + /* XVLDREPL.D encodes its signed displacement in units of 8 bytes. */ + if (!(disp & 7)) { + la_xvldrepl_d(dest, address, disp / 8); + } else { + IR2_OPND src = ra_alloc_ftemp(); + + la_fld_d(src, address, disp); + la_xvreplve0_d(dest, src); + ra_free_temp(src); + } + ra_free_mem_temp(address); + return true; + } + + IR2_OPND src = load_freg128_from_ir1(opnd1); la_xvreplve0_d(dest, src); return true; } @@ -1338,15 +1425,15 @@ bool translate_vextractps(IR1_INST * pir1) { IR1_OPND * opnd2 = ir1_get_opnd(pir1, 2); lsassert(ir1_opnd_is_gpr(opnd0) || ir1_opnd_is_mem(opnd0)); IR2_OPND src = load_freg128_from_ir1(opnd1); - IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND value = ra_alloc_itemp(); uint8_t imm = ir1_opnd_uimm(opnd2) & 0x3; - la_xvpickve_w(temp, src, imm); + la_xvpickve2gr_w(value, src, imm); if (ir1_opnd_is_gpr(opnd0)) { - IR2_OPND dest = ra_alloc_gpr(ir1_opnd_base_reg_num(opnd0)); - la_movfr2gr_d(dest, temp); + store_ireg_to_ir1(value, opnd0, false); } else { - store_freg_to_ir1(temp, opnd0, false, false); + store_u32_to_ir1_mem_exact(value, opnd0); } + ra_free_temp(value); return true; } @@ -1441,10 +1528,14 @@ bool translate_vinserti128(IR1_INST * pir1) { ir1_opnd_is_ymm(ir1_get_opnd(pir1, 1)) && ir1_opnd_size(ir1_get_opnd(pir1, 2)) == 128 && ir1_opnd_is_imm(ir1_get_opnd(pir1, 3))); - IR2_OPND dest = load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); - IR2_OPND src1 = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); uint8 imm = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)) & 0x1; + IR2_OPND dest = imm ? + ra_alloc_xmm(ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0))) : + load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); + IR2_OPND src1 = imm ? + ra_alloc_xmm(ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) : + load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); + IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); bool dest_is_src1 = ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) == ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1)); bool dest_is_src2 = ir1_opnd_is_xmm(ir1_get_opnd(pir1, 2)) && @@ -1471,25 +1562,7 @@ bool translate_vinsertf128(IR1_INST * pir1) { if (!option_enable_lasx) { return translate_vinsertf128_lsx(pir1); } - - lsassert(ir1_opnd_num(pir1) == 4); - lsassert(ir1_opnd_is_ymm(ir1_get_opnd(pir1, 0)) && - ir1_opnd_is_ymm(ir1_get_opnd(pir1, 1)) && - ir1_opnd_size(ir1_get_opnd(pir1, 2)) == 128 && - ir1_opnd_is_imm(ir1_get_opnd(pir1, 3))); - IR2_OPND dest = load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); - IR2_OPND src1 = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - uint8 imm = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)) & 0x1; - IR2_OPND temp = ra_alloc_ftemp(); - la_xvori_b(temp, src1, 0); - if (!imm) { - la_xvpermi_q(temp, src2, VEXTRINS_IMM_4_0(3, 0)); - } else { - la_xvpermi_q(temp, src2, VEXTRINS_IMM_4_0(0, 2)); - } - la_xvori_b(dest, temp, 0); - return true; + return translate_vinserti128(pir1); } bool translate_vshufpd(IR1_INST * pir1) { @@ -1506,9 +1579,30 @@ bool translate_vshufpd(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp_l = ra_alloc_ftemp(); - IR2_OPND temp_h = ra_alloc_ftemp(); uint8_t imm8 = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)) & 0xf; + + if ((imm8 & 0x5) == ((imm8 >> 1) & 0x5)) { + if (imm8 == 0) { + la_xvilvl_d(dest, src2, src1); + } else if (imm8 == 0xf) { + la_xvilvh_d(dest, src2, src1); + } else { + IR2_OPND low = ra_alloc_ftemp(); + + if (imm8 == 0xc) { + la_xvilvl_d(low, src2, src1); + la_xvilvh_d(dest, src2, src1); + } else { + la_xvilvh_d(low, src2, src1); + la_xvilvl_d(dest, src2, src1); + } + la_xvpermi_q(dest, low, VEXTRINS_IMM_4_0(3, 0)); + } + return true; + } + IR2_OPND temp_l = ra_alloc_ftemp(); + IR2_OPND result = ir2_opnd_cmp(&dest, &src2) ? + ra_alloc_ftemp() : dest; uint8_t l = imm8 & 0x3; uint8_t h = imm8 >> 2; if (l == 0) { @@ -1535,27 +1629,27 @@ bool translate_vshufpd(IR1_INST * pir1) { lsassert(0); } la_xvori_b(temp_l, src1, 0); - la_xvori_b(temp_h, src1, 0); la_xvshuf4i_d(temp_l, src2, l); - la_xvshuf4i_d(temp_h, src2, h); - la_xvpermi_q(temp_h, temp_l, VEXTRINS_IMM_4_0(3, 0)); - la_xvori_b(dest, temp_h, 0); + if (!ir2_opnd_cmp(&result, &src1)) + la_xvori_b(result, src1, 0); + la_xvshuf4i_d(result, src2, h); + la_xvpermi_q(result, temp_l, VEXTRINS_IMM_4_0(3, 0)); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - bool same_sources = ir1_opnd_is_xmm(ir1_get_opnd(pir1, 2)) && - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1)) == - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 2)); - bool dest_is_src1 = ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) == - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1)); - bool dest_is_src2 = ir1_opnd_is_xmm(ir1_get_opnd(pir1, 2)) && - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) == - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 2)); - bool write_dest = dest_is_src1 || !dest_is_src2; - IR2_OPND temp = write_dest ? dest : ra_alloc_ftemp(); + IR2_OPND result = dest; uint8_t imm8 = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)); imm8 = imm8 & 3; + if (ir2_opnd_cmp(&src1, &src2)) { + static const uint8_t permi[4] = {0x44, 0x4e, 0xe4, 0xee}; + + la_vshuf4i_w(dest, src1, permi[imm8]); + set_high128_xreg_to_zero(dest); + return true; + } if (imm8 == 0) { imm8 = 0x8; } else if (imm8 == 1) { @@ -1567,21 +1661,13 @@ bool translate_vshufpd(IR1_INST * pir1) { } else { lsassert(0); } - if (same_sources) { - static const uint8_t permi[4] = {0x44, 0x4e, 0xe4, 0xee}; - - la_vshuf4i_w(dest, src1, permi[ - ir1_opnd_uimm(ir1_get_opnd(pir1, 3)) & 3]); - set_high128_xreg_to_zero(dest); - return true; - } - if (!dest_is_src1) { - la_vori_b(temp, src1, 0); - } - la_vshuf4i_d(temp, src2, imm8); - if (!write_dest) { - la_vori_b(dest, temp, 0); - } + if (ir2_opnd_cmp(&dest, &src2) && !ir2_opnd_cmp(&dest, &src1)) + result = ra_alloc_ftemp(); + if (!ir2_opnd_cmp(&result, &src1)) + la_vori_b(result, src1, 0); + la_vshuf4i_d(result, src2, imm8); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); set_high128_xreg_to_zero(dest); } return true; @@ -1629,8 +1715,7 @@ bool translate_vshufps(IR1_INST * pir1) { IR2_OPND temp2 = ra_alloc_ftemp(); la_vshuf4i_w(temp1, src1, imm8); la_vshuf4i_w(temp2, src2, imm8 >> 4); - la_vpickev_d(temp1, temp2, temp1); - la_xvori_b(dest, temp1, 0); + la_vpickev_d(dest, temp2, temp1); set_high128_xreg_to_zero(dest); } return true; @@ -1650,20 +1735,14 @@ bool translate_vunpckhpd(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvori_b(temp, src1, 0); - la_xvshuf4i_d(temp, src2, 0xd); - la_xvori_b(dest, temp, 0); + la_xvilvh_d(dest, src2, src1); } else { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vori_b(temp, src1, 0); - la_vshuf4i_d(temp, src2, 0xd); - la_xvori_b(dest, temp, 0); + la_xvilvh_d(dest, src2, src1); set_high128_xreg_to_zero(dest); } return true; @@ -1689,10 +1768,8 @@ bool translate_vunpckhps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vilvh_w(temp, src2, src1); - la_xvori_b(dest, temp, 0); + la_vilvh_w(dest, src2, src1); set_high128_xreg_to_zero(dest); } return true; @@ -1712,20 +1789,14 @@ bool translate_vunpcklpd(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvori_b(temp, src1, 0); - la_xvshuf4i_d(temp, src2, 0x8); - la_xvori_b(dest, temp, 0); + la_xvilvl_d(dest, src2, src1); } else { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vori_b(temp, src1, 0); - la_vshuf4i_d(temp, src2, 0x8); - la_xvori_b(dest, temp, 0); + la_xvilvl_d(dest, src2, src1); set_high128_xreg_to_zero(dest); } return true; @@ -1751,10 +1822,8 @@ bool translate_vunpcklps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vilvl_w(temp, src2, src1); - la_xvori_b(dest, temp, 0); + la_vilvl_w(dest, src2, src1); set_high128_xreg_to_zero(dest); } return true; @@ -1771,7 +1840,6 @@ bool translate_vpabsx(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src = load_freg256_from_ir1(opnd1); IR2_OPND vzero = ra_alloc_ftemp(); - IR2_OPND temp = ra_alloc_ftemp(); IR2_INST * ( * tr_inst_128)(IR2_OPND, IR2_OPND, IR2_OPND); IR2_INST * ( * tr_inst_256)(IR2_OPND, IR2_OPND, IR2_OPND); switch (ir1_opcode(pir1)) { @@ -1796,10 +1864,9 @@ bool translate_vpabsx(IR1_INST * pir1) { la_vreplgr2vr_d(vzero, zero_ir2_opnd); if (ir1_opnd_is_ymm(opnd0)) - tr_inst_256(temp, src, vzero); + tr_inst_256(dest, src, vzero); else - tr_inst_128(temp, src, vzero); - la_xvori_b(dest, temp, 0); + tr_inst_128(dest, src, vzero); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); @@ -1960,9 +2027,7 @@ bool translate_vpandn(IR1_INST * pir1) { la_xvandn_v(dest, src1, src2); return true; } - IR2_OPND temp = ra_alloc_ftemp(); - la_xvandn_v(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vandn_v(dest, src1, src2); set_high128_xreg_to_zero(dest); return true; } @@ -1983,30 +2048,33 @@ bool translate_vpblendd(IR1_INST * pir1) { IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(opnd2); uint8_t imm = ir1_opnd_uimm(opnd3); - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND rmask = ra_alloc_itemp(); if (ir1_opnd_is_xmm(opnd0)) { + bool dest_is_src1 = ir1_opnd_is_same_reg(opnd0, opnd1); + bool dest_is_src2 = ir1_opnd_is_same_reg(opnd0, opnd2); if((imm & 0xf) == 0xf) { - la_xvori_b(dest, src2, 0); + if (!dest_is_src2) + la_xvori_b(dest, src2, 0); set_high128_xreg_to_zero(dest); return true; } else if((imm & 0xf) == 0x0){ - la_xvori_b(dest, src1, 0); + if (!dest_is_src1) + la_xvori_b(dest, src1, 0); set_high128_xreg_to_zero(dest); return true; } - uint64_t mask = 0; + bool base_is_src2 = dest_is_src2 || + (!dest_is_src1 && __builtin_popcount(imm & 0xf) > 2); + uint8_t insert_mask = (base_is_src2 ? ~imm : imm) & 0xf; + IR2_OPND base = base_is_src2 ? src2 : src1; + IR2_OPND insert_src = base_is_src2 ? src1 : src2; + + if (!dest_is_src1 && !dest_is_src2) + la_xvori_b(dest, base, 0); for (int i = 0; i < 4; i++) { - if (imm & (1 << i)) { - mask |= (0xFFULL) << (i * 8); - } + if (insert_mask & (1u << i)) + la_vextrins_w(dest, insert_src, VEXTRINS_IMM_4_0(i, i)); } - li_d(rmask, mask); - la_movgr2fr_d(temp, rmask); - la_vext2xv_w_b(temp, temp); - la_xvbitsel_v(dest, src1, src2, temp); - set_high128_xreg_to_zero(dest); return true; } @@ -2017,6 +2085,8 @@ bool translate_vpblendd(IR1_INST * pir1) { la_xvori_b(dest, src1, 0); return true; } + IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND rmask = ra_alloc_itemp(); uint64_t mask = 0; for (int i = 0; i < 8; i++) { if (imm & (1 << i)) { @@ -2171,18 +2241,27 @@ bool translate_vperm2i128(IR1_INST * pir1) { } IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvori_b(temp, src2, 0); - la_xvpermi_q(temp, src1, imm); + IR2_OPND result = dest; + + if (ir2_opnd_cmp(&src1, &src2) && (imm & 0xbb) == 0) { + la_xvreplve0_q(dest, src1); + return true; + } + if (ir2_opnd_cmp(&dest, &src1) && !ir2_opnd_cmp(&src1, &src2)) + result = ra_alloc_ftemp(); + if (!ir2_opnd_cmp(&result, &src2)) + la_xvori_b(result, src2, 0); + la_xvpermi_q(result, src1, imm); if (zero_l) { - la_xvinsgr2vr_d(temp, zero_ir2_opnd, 0); - la_xvinsgr2vr_d(temp, zero_ir2_opnd, 1); + la_xvinsgr2vr_d(result, zero_ir2_opnd, 0); + la_xvinsgr2vr_d(result, zero_ir2_opnd, 1); } if (zero_h) { - la_xvinsgr2vr_d(temp, zero_ir2_opnd, 2); - la_xvinsgr2vr_d(temp, zero_ir2_opnd, 3); + la_xvinsgr2vr_d(result, zero_ir2_opnd, 2); + la_xvinsgr2vr_d(result, zero_ir2_opnd, 3); } - la_xvori_b(dest, temp, 0); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); return true; } @@ -2519,52 +2598,92 @@ bool translate_vpsubx(IR1_INST * pir1) { return true; } -bool translate_vptest(IR1_INST * pir1) { - if (!option_enable_lasx) { - return translate_vptest_lsx(pir1); - } +/* Requested ZF/CF values also remain in fcc0/fcc1 for a fused branch. */ +void generate_avx_test_flags(IR1_INST *pir1, IR2_OPND dest, IR2_OPND src, + IR2_OPND flags, uint8_t eflags) +{ + bool wide = ir1_opnd_is_ymm(ir1_get_opnd(pir1, 0)); + int shift = ir1_opcode(pir1) == dt_X86_INS_VTESTPS ? 31 : + ir1_opcode(pir1) == dt_X86_INS_VTESTPD ? 63 : 0; - IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); - IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); - lsassert(ir1_opnd_is_xmm(opnd0) || ir1_opnd_is_ymm(opnd0)); - IR2_OPND dest = load_freg256_from_ir1(opnd0); - IR2_OPND src = load_freg256_from_ir1(opnd1); + if (!(eflags & CF_USEDEF_BIT)) { + la_mov64(flags, zero_ir2_opnd); + } + if (!(eflags & (ZF_USEDEF_BIT | CF_USEDEF_BIT))) { + return; + } IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND label_1 = ra_alloc_label(); - IR2_OPND label_2 = ra_alloc_label(); - IR2_OPND n4095_opnd = ra_alloc_num_4095(); - - if (ir1_opnd_is_xmm(opnd0)) { - la_x86mtflag(zero_ir2_opnd, 0x3f); - la_vand_v(temp, src, dest); - la_vseteqz_v(fcc0_ir2_opnd, temp); - la_bceqz(fcc0_ir2_opnd, label_1); - la_x86mtflag(n4095_opnd, ZF_USEDEF_BIT); - - la_label(label_1); - la_vandn_v(temp, dest, src); - la_vseteqz_v(fcc0_ir2_opnd, temp); - la_bceqz(fcc0_ir2_opnd, label_2); - la_x86mtflag(n4095_opnd, CF_USEDEF_BIT); - la_label(label_2); - } else { - la_x86mtflag(zero_ir2_opnd, 0x3f); - la_xvand_v(temp, src, dest); - la_xvseteqz_v(fcc0_ir2_opnd, temp); - la_bceqz(fcc0_ir2_opnd, label_1); - la_x86mtflag(n4095_opnd, ZF_USEDEF_BIT); + IR2_OPND bit = ra_alloc_itemp(); + for (int cf = 1; cf >= 0; cf--) { + if (!(eflags & (cf ? CF_USEDEF_BIT : ZF_USEDEF_BIT))) { + continue; + } + IR2_OPND condition = cf ? fcc1_ir2_opnd : fcc0_ir2_opnd; + if (wide) { + if (cf) { + la_xvandn_v(temp, dest, src); + } else { + la_xvand_v(temp, dest, src); + } + if (shift == 31) { + la_xvsrli_w(temp, temp, shift); + } else if (shift == 63) { + la_xvsrli_d(temp, temp, shift); + } + la_xvseteqz_v(condition, temp); + } else { + if (cf) { + la_vandn_v(temp, dest, src); + } else { + la_vand_v(temp, dest, src); + } + if (shift == 31) { + la_vsrli_w(temp, temp, shift); + } else if (shift == 63) { + la_vsrli_d(temp, temp, shift); + } + la_vseteqz_v(condition, temp); + } + if (cf) { + la_movcf2gr(flags, condition); + } else { + la_movcf2gr(bit, condition); + la_bstrins_w(flags, bit, ZF_BIT_INDEX, ZF_BIT_INDEX); + } + } + ra_free_temp(bit); + ra_free_temp(temp); +} - la_label(label_1); - la_xvandn_v(temp, dest, src); - la_xvseteqz_v(fcc0_ir2_opnd, temp); - la_bceqz(fcc0_ir2_opnd, label_2); - la_x86mtflag(n4095_opnd, CF_USEDEF_BIT); - la_label(label_2); +static bool translate_avx_test(IR1_INST *pir1) +{ + IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0); + IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); + bool wide = ir1_opnd_is_ymm(opnd0); + IR2_OPND dest = wide ? load_freg256_from_ir1(opnd0) : + load_freg128_from_ir1(opnd0); + IR2_OPND src = wide ? load_freg256_from_ir1(opnd1) : + load_freg128_from_ir1(opnd1); + uint8_t eflags = ir1_get_eflag_def(pir1) & + ~lsenv->tr_data->curr_ir1_skipped_eflags; + /* Operand loads must still fault when all generated flags are dead. */ + if (eflags) { + IR2_OPND flags = ra_alloc_itemp(); + generate_avx_test_flags(pir1, dest, src, flags, eflags); + la_x86mtflag(flags, eflags); + ra_free_temp(flags); } - ra_free_num_4095(n4095_opnd); return true; } +bool translate_vptest(IR1_INST *pir1) +{ + if (!option_enable_lasx) { + return translate_vptest_lsx(pir1); + } + return translate_avx_test(pir1); +} + bool translate_vpunpckhxx(IR1_INST * pir1) { if (!option_enable_lasx) { return translate_vpunpckhxx_lsx(pir1); @@ -3110,18 +3229,23 @@ bool translate_vfnmaddxxxpd(IR1_INST * pir1) { IR2_OPND src3_temp = ra_alloc_ftemp(); /* check if result is NaN */ - /* Unused upper lanes must not raise floating-point exceptions. */ + IR2_OPND label_over = ra_alloc_label(); if (ir1_opnd_is_xmm(opnd0)) { la_vfcmp_cond_d(mask, temp, temp, 0x8); + la_vseteqz_v(fcc0_ir2_opnd, mask); } else { la_xvfcmp_cond_d(mask, temp, temp, 0x8); + la_xvseteqz_v(fcc0_ir2_opnd, mask); } + la_bcnez(fcc0_ir2_opnd, label_over); + la_xvand_v(src1_temp, mask, dest); la_xvand_v(src2_temp, mask, src1); la_xvand_v(src3_temp, mask, src2); tr_inst(src1_temp, src2_temp, src3_temp, src1_temp); la_xvbitsel_v(temp, temp, src1_temp, mask); + la_label(label_over); la_xvori_b(dest, temp, 0); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); @@ -3184,18 +3308,23 @@ bool translate_vfnmaddxxxps(IR1_INST * pir1) { IR2_OPND src3_temp = ra_alloc_ftemp(); /* check if result is NaN */ - /* Unused upper lanes must not raise floating-point exceptions. */ + IR2_OPND label_over = ra_alloc_label(); if (ir1_opnd_is_xmm(opnd0)) { la_vfcmp_cond_s(mask, temp, temp, 0x8); + la_vseteqz_v(fcc0_ir2_opnd, mask); } else { la_xvfcmp_cond_s(mask, temp, temp, 0x8); + la_xvseteqz_v(fcc0_ir2_opnd, mask); } + la_bcnez(fcc0_ir2_opnd, label_over); + la_xvand_v(src1_temp, mask, dest); la_xvand_v(src2_temp, mask, src1); la_xvand_v(src3_temp, mask, src2); tr_inst(src1_temp, src2_temp, src3_temp, src1_temp); la_xvbitsel_v(temp, temp, src1_temp, mask); + la_label(label_over); la_xvori_b(dest, temp, 0); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); @@ -3353,18 +3482,23 @@ bool translate_vfnmsubxxxpd(IR1_INST * pir1) { IR2_OPND src3_temp = ra_alloc_ftemp(); /* check if result is NaN */ - /* Unused upper lanes must not raise floating-point exceptions. */ + IR2_OPND label_over = ra_alloc_label(); if (ir1_opnd_is_xmm(opnd0)) { la_vfcmp_cond_d(mask, temp, temp, 0x8); + la_vseteqz_v(fcc0_ir2_opnd, mask); } else { la_xvfcmp_cond_d(mask, temp, temp, 0x8); + la_xvseteqz_v(fcc0_ir2_opnd, mask); } + la_bcnez(fcc0_ir2_opnd, label_over); + la_xvand_v(src1_temp, mask, dest); la_xvand_v(src2_temp, mask, src1); la_xvand_v(src3_temp, mask, src2); tr_inst(src1_temp, src2_temp, src3_temp, src1_temp); la_xvbitsel_v(temp, temp, src1_temp, mask); + la_label(label_over); la_xvori_b(dest, temp, 0); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); @@ -3424,18 +3558,23 @@ bool translate_vfnmsubxxxps(IR1_INST * pir1) { IR2_OPND src3_temp = ra_alloc_ftemp(); /* check if result is NaN */ - /* Unused upper lanes must not raise floating-point exceptions. */ + IR2_OPND label_over = ra_alloc_label(); if (ir1_opnd_is_xmm(opnd0)) { la_vfcmp_cond_s(mask, temp, temp, 0x8); + la_vseteqz_v(fcc0_ir2_opnd, mask); } else { la_xvfcmp_cond_s(mask, temp, temp, 0x8); + la_xvseteqz_v(fcc0_ir2_opnd, mask); } + la_bcnez(fcc0_ir2_opnd, label_over); + la_xvand_v(src1_temp, mask, dest); la_xvand_v(src2_temp, mask, src1); la_xvand_v(src3_temp, mask, src2); tr_inst(src1_temp, src2_temp, src3_temp, src1_temp); la_xvbitsel_v(temp, temp, src1_temp, mask); + la_label(label_over); la_xvori_b(dest, temp, 0); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); @@ -3471,8 +3610,6 @@ bool translate_vpbroadcastq(IR1_INST *pir1) set_high128_xreg_to_zero(dest); } else if (ir1_opnd_is_ymm(opnd0)) { la_xvreplve0_d(dest, src); - la_xvinsve0_d(dest, dest, 2); - la_xvinsve0_d(dest, dest, 3); } else { lsassert(0); } @@ -3707,7 +3844,8 @@ bool translate_vrsqrtss(IR1_INST * pir1) { /* this x86 instruction has no exception ,so we need mask all exception */ la_frsqrt_s(temp, src2); - la_xvori_b(dest, src1, 0x0); + if (!ir2_opnd_cmp(&dest, &src1)) + la_xvori_b(dest, src1, 0x0); la_xvinsve0_w(dest, temp, 0); set_high128_xreg_to_zero(dest); @@ -3768,7 +3906,8 @@ bool translate_vrcpss(IR1_INST * pir1) { /* this x86 instruction has no exception ,so we need mask all exception */ la_frecip_s(temp, src2); - la_xvori_b(dest, src1, 0x0); + if (!ir2_opnd_cmp(&dest, &src1)) + la_xvori_b(dest, src1, 0x0); la_xvinsve0_w(dest, temp, 0); set_high128_xreg_to_zero(dest); @@ -3980,32 +4119,17 @@ bool translate_vphsubw(IR1_INST * pir1) { IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); IR1_OPND * opnd2 = ir1_get_opnd(pir1, 2); - if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND dest = load_freg128_from_ir1(opnd0); - IR2_OPND src1 = load_freg128_from_ir1(opnd1); - IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - la_vhsubw_w_h(temp1, src1, src1); - la_vneg_h(temp1, temp1); - la_vhsubw_w_h(temp2, src2, src2); - la_vneg_h(temp2, temp2); + IR2_OPND dest = load_freg256_from_ir1(opnd0); + IR2_OPND src1 = load_freg256_from_ir1(opnd1); + IR2_OPND src2 = load_freg256_from_ir1(opnd2); + IR2_OPND even = ra_alloc_ftemp(); + IR2_OPND odd = ra_alloc_ftemp(); - la_vpickev_h(dest, temp2, temp1); + la_xvpickev_h(even, src2, src1); + la_xvpickod_h(odd, src2, src1); + la_xvsub_h(dest, even, odd); + if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); - } else { - IR2_OPND dest = load_freg256_from_ir1(opnd0); - IR2_OPND src1 = load_freg256_from_ir1(opnd1); - IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - la_xvhsubw_w_h(temp1, src1, src1); - la_xvneg_h(temp1, temp1); - la_xvhsubw_w_h(temp2, src2, src2); - la_xvneg_h(temp2, temp2); - - la_xvpickev_h(dest, temp2, temp1); - } return true; } @@ -4017,32 +4141,17 @@ bool translate_vphsubd(IR1_INST * pir1) { IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); IR1_OPND * opnd2 = ir1_get_opnd(pir1, 2); - if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND dest = load_freg128_from_ir1(opnd0); - IR2_OPND src1 = load_freg128_from_ir1(opnd1); - IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - la_vhsubw_d_w(temp1, src1, src1); - la_vneg_w(temp1, temp1); - la_vhsubw_d_w(temp2, src2, src2); - la_vneg_w(temp2, temp2); + IR2_OPND dest = load_freg256_from_ir1(opnd0); + IR2_OPND src1 = load_freg256_from_ir1(opnd1); + IR2_OPND src2 = load_freg256_from_ir1(opnd2); + IR2_OPND even = ra_alloc_ftemp(); + IR2_OPND odd = ra_alloc_ftemp(); - la_vpickev_w(dest, temp2, temp1); + la_xvpickev_w(even, src2, src1); + la_xvpickod_w(odd, src2, src1); + la_xvsub_w(dest, even, odd); + if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); - } else { - IR2_OPND dest = load_freg256_from_ir1(opnd0); - IR2_OPND src1 = load_freg256_from_ir1(opnd1); - IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - la_xvhsubw_d_w(temp1, src1, src1); - la_xvneg_w(temp1, temp1); - la_xvhsubw_d_w(temp2, src2, src2); - la_xvneg_w(temp2, temp2); - - la_xvpickev_w(dest, temp2, temp1); - } return true; } @@ -4054,45 +4163,17 @@ bool translate_vphsubsw(IR1_INST * pir1) { IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); IR1_OPND * opnd2 = ir1_get_opnd(pir1, 2); - if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND dest = load_freg128_from_ir1(opnd0); - IR2_OPND src1 = load_freg128_from_ir1(opnd1); - IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND temp3 = ra_alloc_ftemp(); - IR2_OPND temp4 = ra_alloc_ftemp(); - - la_vpickev_h(temp1, src1, src1); - la_vpickod_h(temp2, src1, src1); - la_vssub_h(temp3, temp1, temp2); //temp1-temp2 - - la_vpickev_h(temp1, src2, src2); - la_vpickod_h(temp2, src2, src2); - la_vssub_h(temp4, temp1, temp2); //temp1-temp2 + IR2_OPND dest = load_freg256_from_ir1(opnd0); + IR2_OPND src1 = load_freg256_from_ir1(opnd1); + IR2_OPND src2 = load_freg256_from_ir1(opnd2); + IR2_OPND even = ra_alloc_ftemp(); + IR2_OPND odd = ra_alloc_ftemp(); - la_vshuf4i_d(temp4, temp3, 0b00000110); - la_vori_b(dest, temp4, 0x0); + la_xvpickev_h(even, src2, src1); + la_xvpickod_h(odd, src2, src1); + la_xvssub_h(dest, even, odd); + if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); - } else { - IR2_OPND dest = load_freg256_from_ir1(opnd0); - IR2_OPND src1 = load_freg256_from_ir1(opnd1); - IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND temp3 = ra_alloc_ftemp(); - IR2_OPND temp4 = ra_alloc_ftemp(); - la_xvpickev_h(temp1, src1, src1); - la_xvpickod_h(temp2, src1, src1); - la_xvssub_h(temp3, temp1, temp2); //temp1-temp2 - - la_xvpickev_h(temp1, src2, src2); - la_xvpickod_h(temp2, src2, src2); - la_xvssub_h(temp4, temp1, temp2); //temp1-temp2 - - la_xvshuf4i_d(temp4, temp3, 0b00000110); - la_xvori_b(dest, temp4, 0x0); - } return true; } @@ -4368,7 +4449,8 @@ bool translate_vpinsrx(IR1_INST * pir1) { } - la_vori_b(dest, src1, 0x0); + if (!ir2_opnd_cmp(&dest, &src1)) + la_vori_b(dest, src1, 0x0); tr_inst(dest, src2, imm); set_high128_xreg_to_zero(dest); return true; @@ -4388,10 +4470,8 @@ bool translate_vpsllvd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(opnd2); IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); la_vslei_wu(temp, src2, 31); - la_vand_v(temp2, temp, src2); - la_vsll_w(dest, src1, temp2); + la_vsll_w(dest, src1, src2); la_vand_v(dest, dest, temp); set_high128_xreg_to_zero(dest); } else if (ir1_opnd_is_ymm(opnd0)) { @@ -4400,10 +4480,8 @@ bool translate_vpsllvd(IR1_INST * pir1) { IR2_OPND src2 = load_freg256_from_ir1(opnd2); IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); la_xvslei_wu(temp, src2, 31); - la_xvand_v(temp2, temp, src2); - la_xvsll_w(dest, src1, temp2); + la_xvsll_w(dest, src1, src2); la_xvand_v(dest, dest, temp); } @@ -4424,13 +4502,10 @@ bool translate_vpsllvq(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(opnd2); IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - la_vandi_b(temp,temp,0); la_vldi(temp,0b0110000111111); la_vsle_du(temp, src2, temp); - la_vand_v(temp2, temp, src2); - la_vsll_d(dest, src1, temp2); + la_vsll_d(dest, src1, src2); la_vand_v(dest, dest, temp); set_high128_xreg_to_zero(dest); } else if (ir1_opnd_is_ymm(opnd0)) { @@ -4439,13 +4514,10 @@ bool translate_vpsllvq(IR1_INST * pir1) { IR2_OPND src2 = load_freg256_from_ir1(opnd2); IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - la_xvandi_b(temp,temp,0); la_xvldi(temp,0b0110000111111); la_xvsle_du(temp, src2, temp); - la_xvand_v(temp2, temp, src2); - la_xvsll_d(dest, src1, temp2); + la_xvsll_d(dest, src1, src2); la_xvand_v(dest, dest, temp); } @@ -4465,34 +4537,20 @@ bool translate_vpsravd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp01 = ra_alloc_ftemp(); - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND temp1 = ra_alloc_ftemp(); - - la_vclz_w(temp01, src1); - la_vseqi_w(temp01, temp01, 0x0); + IR2_OPND count = ra_alloc_ftemp(); - la_vslei_wu(temp, src2, 31); - la_vand_v(temp1, temp, src2); - la_vsra_w(temp1, src1, temp1); - la_vbitsel_v(dest, temp01, temp1, temp); + la_vmini_wu(count, src2, 31); + la_vsra_w(dest, src1, count); set_high128_xreg_to_zero(dest); } else if (ir1_opnd_is_ymm(opnd0)) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND temp01 = ra_alloc_ftemp(); - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND temp1 = ra_alloc_ftemp(); - - la_xvclz_w(temp01, src1); - la_xvseqi_w(temp01, temp01, 0x0); + IR2_OPND count = ra_alloc_ftemp(); - la_xvslei_wu(temp, src2, 31); - la_xvand_v(temp1, temp, src2); - la_xvsra_w(temp1, src1, temp1); - la_xvbitsel_v(dest, temp01, temp1, temp); + la_xvmini_wu(count, src2, 31); + la_xvsra_w(dest, src1, count); } return true; @@ -4547,10 +4605,15 @@ bool translate_vpermilps(IR1_INST * pir1) { } else { IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); - la_vandi_b(temp, src2, 0b00000011); - la_vshuf_w(temp, src2, src1); - la_vori_b(dest, temp, 0x0); + IR2_OPND result = dest; + + if (ir2_opnd_cmp(&dest, &src1) || + ir2_opnd_cmp(&dest, &src2)) + result = ra_alloc_ftemp(); + la_vandi_b(result, src2, 0b00000011); + la_vshuf_w(result, src2, src1); + if (!ir2_opnd_cmp(&result, &dest)) + la_vori_b(dest, result, 0); } set_high128_xreg_to_zero(dest); } else { @@ -4562,11 +4625,15 @@ bool translate_vpermilps(IR1_INST * pir1) { } else { IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvandi_b(temp, src2, 0b00000011); - la_xvshuf_w(temp, src2, src1); - la_xvori_b(dest, temp, 0x0); - + IR2_OPND result = dest; + + if (ir2_opnd_cmp(&dest, &src1) || + ir2_opnd_cmp(&dest, &src2)) + result = ra_alloc_ftemp(); + la_xvandi_b(result, src2, 0b00000011); + la_xvshuf_w(result, src2, src1); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); } } @@ -4587,17 +4654,25 @@ bool translate_vpermilpd(IR1_INST * pir1) { IR2_OPND src1 = load_freg128_from_ir1(opnd1); if (ir1_opnd_is_imm(opnd2)) { uint8_t imm = ir1_opnd_uimm(opnd2); - imm = 0b10100000 + ((imm & 0b00001000) << 3) + ((imm & 0b00000100) << 2) + ((imm & 0b00000010) << 1) + (imm & 0b00000001); - la_xvpermi_d(dest, src1, imm); + if ((imm & 3) == 0 || (imm & 3) == 3) { + la_vreplvei_d(dest, src1, imm & 1); + } else { + imm = 0b10100000 + ((imm & 0b00001000) << 3) + ((imm & 0b00000100) << 2) + ((imm & 0b00000010) << 1) + (imm & 0b00000001); + la_xvpermi_d(dest, src1, imm); + } } else { IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); - la_vsrli_d(temp, src2, 1); - la_vandi_b(temp, temp, 0b00000001); - la_vshuf_d(temp, src2, src1); - la_vori_b(dest, temp, 0x0); - + IR2_OPND result = dest; + + if (ir2_opnd_cmp(&dest, &src1) || + ir2_opnd_cmp(&dest, &src2)) + result = ra_alloc_ftemp(); + la_vsrli_d(result, src2, 1); + la_vandi_b(result, result, 0b00000001); + la_vshuf_d(result, src2, src1); + if (!ir2_opnd_cmp(&result, &dest)) + la_vori_b(dest, result, 0); } set_high128_xreg_to_zero(dest); } else { @@ -4605,17 +4680,25 @@ bool translate_vpermilpd(IR1_INST * pir1) { IR2_OPND src1 = load_freg256_from_ir1(opnd1); if (ir1_opnd_is_imm(opnd2)) { uint8_t imm = ir1_opnd_uimm(opnd2); + if ((imm & 0xf) == 0 || (imm & 0xf) == 0xf) { + la_xvrepl128vei_d(dest, src1, imm & 1); + return true; + } imm = 0b10100000 + ((imm & 0b00001000) << 3) + ((imm & 0b00000100) << 2) + ((imm & 0b00000010) << 1) + (imm & 0b00000001); la_xvpermi_d(dest, src1, imm); } else { IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvsrli_d(temp, src2, 1); - la_xvandi_b(temp, temp, 0b00000001); - la_xvshuf_d(temp, src2, src1); - la_xvori_b(dest, temp, 0x0); - + IR2_OPND result = dest; + + if (ir2_opnd_cmp(&dest, &src1) || + ir2_opnd_cmp(&dest, &src2)) + result = ra_alloc_ftemp(); + la_xvsrli_d(result, src2, 1); + la_xvandi_b(result, result, 0b00000001); + la_xvshuf_d(result, src2, src1); + if (!ir2_opnd_cmp(&result, &dest)) + la_xvori_b(dest, result, 0); } } @@ -4698,20 +4781,15 @@ bool translate_vphminposuw(IR1_INST * pir1) { IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - la_xvori_b(temp, src, 0x0); - la_vsrli_d(temp, temp, 0x10); - la_vmin_hu(temp, temp, src); - la_vsrli_d(temp, temp, 0x10); - la_vmin_hu(temp, temp, src); - la_vsrli_d(temp, temp, 0x10); - la_vmin_hu(temp, temp, src); - - la_xvpickve_d(temp2, temp, 1); - la_vmin_hu(temp, temp, temp2); + la_vshuf4i_h(temp1, src, 0xb1); + la_vmin_hu(temp, temp1, src); + la_vshuf4i_h(temp1, temp, 0x4e); + la_vmin_hu(temp, temp, temp1); + la_vreplvei_d(temp1, temp, 1); + la_vmin_hu(temp, temp, temp1); la_xvreplve0_h(temp, temp); la_vseq_h(temp1, src, temp); - la_vandi_b(temp2, temp2, 0x0); - la_vfrstp_h(temp2, temp1, temp2); + la_vfrstpi_h(temp2, temp1, 0); la_vpackev_h(dest, temp2, temp); la_xvpickve_w(dest, dest, 0); return true; @@ -4779,119 +4857,14 @@ bool translate_vtestps(IR1_INST * pir1) { if (!option_enable_lasx) { return translate_vtestps_lsx(pir1); } - - IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); - IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); - if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND dest = load_freg128_from_ir1(opnd0); - IR2_OPND src = load_freg128_from_ir1(opnd1); - - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND n4095_opnd = ra_alloc_num_4095(); - IR2_OPND label_1 = ra_alloc_label(); - IR2_OPND label_2 = ra_alloc_label(); - - la_x86mtflag(zero_ir2_opnd, 0x3f); - la_vand_v(temp, dest, src); - la_vsrli_w(temp, temp, 0x1f); - la_vseteqz_v(fcc0_ir2_opnd, temp); //temp==0 fcc=1; - la_bceqz(fcc0_ir2_opnd, label_1); //fcc==1,not jump - la_x86mtflag(n4095_opnd, ZF_USEDEF_BIT); - - la_label(label_1); - la_vandn_v(temp, dest, src); - la_vsrli_w(temp, temp, 0x1f); - la_vseteqz_v(fcc0_ir2_opnd, temp); //temp==0 fcc=1; - la_bceqz(fcc0_ir2_opnd, label_2); //fcc==1,not jump - la_x86mtflag(n4095_opnd, CF_USEDEF_BIT); - la_label(label_2); - - } else { - IR2_OPND dest = load_freg256_from_ir1(opnd0); - IR2_OPND src = load_freg256_from_ir1(opnd1); - - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND n4095_opnd = ra_alloc_num_4095(); - IR2_OPND label_1 = ra_alloc_label(); - IR2_OPND label_2 = ra_alloc_label(); - - la_x86mtflag(zero_ir2_opnd, 0x3f); - la_xvand_v(temp, dest, src); - la_xvsrli_w(temp, temp, 0x1f); - la_xvseteqz_v(fcc0_ir2_opnd, temp); //temp==0 fcc=1; - la_bceqz(fcc0_ir2_opnd, label_1); //fcc==1,not jump - la_x86mtflag(n4095_opnd, ZF_USEDEF_BIT); - - la_label(label_1); - la_xvandn_v(temp, dest, src); - la_xvsrli_w(temp, temp, 0x1f); - la_xvseteqz_v(fcc0_ir2_opnd, temp); //temp==0 fcc=1; - la_bceqz(fcc0_ir2_opnd, label_2); //fcc==1,not jump - la_x86mtflag(n4095_opnd, CF_USEDEF_BIT); - la_label(label_2); - } - - return true; + return translate_avx_test(pir1); } bool translate_vtestpd(IR1_INST * pir1) { if (!option_enable_lasx) { return translate_vtestpd_lsx(pir1); } - - IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); - IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); - if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND dest = load_freg128_from_ir1(opnd0); - IR2_OPND src = load_freg128_from_ir1(opnd1); - - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND n4095_opnd = ra_alloc_num_4095(); - IR2_OPND label_1 = ra_alloc_label(); - IR2_OPND label_2 = ra_alloc_label(); - - la_x86mtflag(zero_ir2_opnd, 0x3f); - la_vand_v(temp, dest, src); - la_vsrli_d(temp, temp, 0x3f); - la_vseteqz_v(fcc0_ir2_opnd, temp); //temp==0 fcc=1; - la_bceqz(fcc0_ir2_opnd, label_1); //fcc==1,not jump - la_x86mtflag(n4095_opnd, ZF_USEDEF_BIT); - - la_label(label_1); - la_vandn_v(temp, dest, src); - la_vsrli_d(temp, temp, 0x3f); - la_vseteqz_v(fcc0_ir2_opnd, temp); //temp==0 fcc=1; - la_bceqz(fcc0_ir2_opnd, label_2); //fcc==1,not jump - la_x86mtflag(n4095_opnd, CF_USEDEF_BIT); - la_label(label_2); - - } else { - IR2_OPND dest = load_freg256_from_ir1(opnd0); - IR2_OPND src = load_freg256_from_ir1(opnd1); - - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND n4095_opnd = ra_alloc_num_4095(); - IR2_OPND label_1 = ra_alloc_label(); - IR2_OPND label_2 = ra_alloc_label(); - - la_x86mtflag(zero_ir2_opnd, 0x3f); - la_xvand_v(temp, dest, src); - la_xvsrli_d(temp, temp, 0x3f); - la_xvseteqz_v(fcc0_ir2_opnd, temp); //temp==0 fcc=1; - la_bceqz(fcc0_ir2_opnd, label_1); //fcc==1,not jump - la_x86mtflag(n4095_opnd, ZF_USEDEF_BIT); - - la_label(label_1); - la_xvandn_v(temp, dest, src); - la_xvsrli_d(temp, temp, 0x3f); - la_xvseteqz_v(fcc0_ir2_opnd, temp); //temp==0 fcc=1; - la_bceqz(fcc0_ir2_opnd, label_2); //fcc==1,not jump - la_x86mtflag(n4095_opnd, CF_USEDEF_BIT); - la_label(label_2); - - } - - return true; + return translate_avx_test(pir1); } bool translate_vpackssxx(IR1_INST * pir1) { @@ -5189,7 +5162,6 @@ bool translate_vmpsadbw(IR1_INST *pir1) IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); IR2_OPND temp3 = ra_alloc_ftemp(); - IR2_OPND temp_dest = ra_alloc_ftemp(); la_vreplvei_w(temp1, src2, imm & 3); la_vmepatmsk_v(temp2, 1, imm & 0x4); @@ -5201,13 +5173,15 @@ bool translate_vmpsadbw(IR1_INST *pir1) la_vhaddw_hu_bu(temp2, temp2, temp2); la_vhaddw_wu_hu(temp2, temp2, temp2); la_vhaddw_hu_bu(temp3, temp3, temp3); - la_vhaddw_wu_hu(temp_dest, temp3, temp3); - la_vsrlni_h_w(temp_dest, temp2, 0); if(ir1_opnd_is_xmm(opnd0)){ - la_xvori_b(dest, temp_dest, 0); + la_vhaddw_wu_hu(dest, temp3, temp3); + la_vsrlni_h_w(dest, temp2, 0); set_high128_xreg_to_zero(dest); return true; } + IR2_OPND temp_dest = ra_alloc_ftemp(); + la_vhaddw_wu_hu(temp_dest, temp3, temp3); + la_vsrlni_h_w(temp_dest, temp2, 0); imm = imm>>3; la_vmepatmsk_v(temp2, 1, imm & 0x4); la_vmepatmsk_v(temp3, 2, imm & 0x4); @@ -5621,6 +5595,52 @@ bool translate_vroundss(IR1_INST *pir1) IR2_OPND mxcsr = ra_alloc_itemp(); bool is_xmm = ir1_opnd_is_xmm(opnd0); la_xvreplve0_w(src, src2); + IR2_OPND merge = ra_alloc_label(); + if (!(imm & 0x4)) { + IR2_OPND slow = ra_alloc_label(); + IR2_OPND sticky = ra_alloc_label(); + IR2_INST *(*fixed_round)(IR2_OPND, IR2_OPND); + + switch (imm & 0x3) { + case 0: + fixed_round = la_vfrintrne_s; + break; + case 1: + fixed_round = la_vfrintrm_s; + break; + case 2: + fixed_round = la_vfrintrp_s; + break; + default: + fixed_round = la_vfrintrz_s; + break; + } + + la_movfcsr2gr(fcsr, fcsr_ir2_opnd); + la_andi(fcsr_save, fcsr, FCSR_ENABLE_SET); + la_bnez(fcsr_save, slow); + if (imm & 0x8) { + la_bstrpick_w(fcsr_save, fcsr, FCSR_OFF_FLAGS_I, + FCSR_OFF_FLAGS_I); + la_bnez(fcsr_save, sticky); + /* Keep invalid status, but discard the round's inexact status. */ + la_vfcmp_cond_s(temp, src, src, 0x8); + la_movfcsr2gr(fcsr, fcsr_ir2_opnd); + } + fixed_round(temp_dest, src); + if (imm & 0x8) { + la_movgr2fcsr(fcsr_ir2_opnd, fcsr); + } + la_b(merge); + if (imm & 0x8) { + la_label(sticky); + /* Inexact is already sticky; the compare restores final causes. */ + fixed_round(temp_dest, src); + la_vfcmp_cond_s(temp, src, src, 0x8); + la_b(merge); + } + la_label(slow); + } if(imm & 0x8){ IR2_INST * ( * tr_inst)(IR2_OPND, IR2_OPND, IR2_OPND, int); tr_inst = is_xmm ? la_vfcmp_cond_s : la_xvfcmp_cond_s; @@ -5683,13 +5703,14 @@ bool translate_vroundss(IR1_INST *pir1) tr_inst = is_xmm ? la_vfrintrz_s : la_xvfrintrz_s; tr_inst(temp_dest, src); } + la_movgr2fcsr(fcsr_ir2_opnd, fcsr_save); + la_label(merge); if (ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) { la_xvori_b(dest, src1, 0); } la_xvinsve0_w(dest, temp_dest, 0); set_high128_xreg_to_zero(dest); - la_movgr2fcsr(fcsr_ir2_opnd, fcsr_save); ra_free_temp(temp); ra_free_temp(fcsr); @@ -5722,6 +5743,31 @@ bool translate_vroundsd(IR1_INST *pir1) IR2_OPND mxcsr = ra_alloc_itemp(); bool is_xmm = ir1_opnd_is_xmm(opnd0); la_xvreplve0_d(src, src2); + IR2_OPND merge = ra_alloc_label(); + if (!(imm & 0xc)) { + IR2_OPND slow = ra_alloc_label(); + + /* With traps masked, the fixed round also supplies exception flags. */ + la_movfcsr2gr(fcsr, fcsr_ir2_opnd); + la_andi(fcsr, fcsr, FCSR_ENABLE_SET); + la_bnez(fcsr, slow); + switch (imm & 0x3) { + case 0: + la_vfrintrne_d(temp_dest, src); + break; + case 1: + la_vfrintrm_d(temp_dest, src); + break; + case 2: + la_vfrintrp_d(temp_dest, src); + break; + default: + la_vfrintrz_d(temp_dest, src); + break; + } + la_b(merge); + la_label(slow); + } if(imm & 0x8){ IR2_INST * ( * tr_inst)(IR2_OPND, IR2_OPND, IR2_OPND, int); tr_inst = is_xmm ? la_vfcmp_cond_d : la_xvfcmp_cond_d; @@ -5784,13 +5830,14 @@ bool translate_vroundsd(IR1_INST *pir1) tr_inst = is_xmm ? la_vfrintrz_d : la_xvfrintrz_d; tr_inst(temp_dest, src); } + la_movgr2fcsr(fcsr_ir2_opnd, fcsr_save); + la_label(merge); if (ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) { la_xvori_b(dest, src1, 0); } la_xvinsve0_d(dest, temp_dest, 0); set_high128_xreg_to_zero(dest); - la_movgr2fcsr(fcsr_ir2_opnd, fcsr_save); ra_free_temp(temp); ra_free_temp(fcsr); @@ -5927,7 +5974,6 @@ bool translate_vpclmulqdq(IR1_INST * pir1) { } else { IR2_OPND src1 = ra_alloc_xmm(s1); IR2_OPND src2; - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND ftemp = ra_alloc_ftemp(); if (!ir1_opnd_is_mem(opnd2)) { @@ -5953,13 +5999,11 @@ bool translate_vpclmulqdq(IR1_INST * pir1) { ra_free_temp(ctrlp); emit_pclmul_ctz_loop(lhs, rhs, res_lo, res_hi); - la_vxor_v(temp, temp, temp); - la_vinsgr2vr_d(temp, res_lo, 0); - la_vinsgr2vr_d(temp, res_hi, 1); - la_xvori_b(dest, temp, 0); + la_vxor_v(dest, dest, dest); + la_vinsgr2vr_d(dest, res_lo, 0); + la_vinsgr2vr_d(dest, res_hi, 1); set_high128_xreg_to_zero(dest); ra_free_temp_auto(src2); - ra_free_temp(temp); ra_free_temp(ftemp); ra_free_temp(lhs); ra_free_temp(rhs); diff --git a/target/i386/latx/translator/tr-mov.c b/target/i386/latx/translator/tr-mov.c index b1c8abd097..e9918dc108 100644 --- a/target/i386/latx/translator/tr-mov.c +++ b/target/i386/latx/translator/tr-mov.c @@ -2142,52 +2142,22 @@ bool translate_popal(IR1_INST *pir1) { return true; } -/* -* int hacker_popcnt(uint32_t n) -* { -* n -= (n>>1) & 0x55555555; -* n = (n & 0x33333333) + ((n>>2) & 0x33333333); -* n = ((n>>4) + n) & 0x0F0F0F0F; -* n += n>>8; -* n += n>>16; -* return n & 0x0000003F; -* } -*/ bool translate_popcnt(IR1_INST *pir1) { IR2_OPND src_opnd = load_ireg_from_ir1(ir1_get_opnd(pir1, 1), ZERO_EXTENSION, false); IR2_OPND count = ra_alloc_itemp(); - IR2_OPND temp = ra_alloc_itemp(); - IR2_OPND src_temp = ra_alloc_itemp(); - IR2_OPND pop_count_ir2_opnd = ra_alloc_itemp(); - int pop_count = ir1_opnd_size(ir1_get_opnd(pir1, 0)); + IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND label_exit = ra_alloc_label(); - IR2_OPND label_zero = ra_alloc_label(); - IR2_OPND label_start = ra_alloc_label(); - - /* set O S Z A C P = 0 */ - la_x86mtflag(zero_ir2_opnd, 0x3f); - la_ori(count, zero_ir2_opnd, 0); - - la_beq(src_opnd, zero_ir2_opnd, label_zero); - - li_d(pop_count_ir2_opnd, pop_count); - - la_or(src_temp, zero_ir2_opnd, src_opnd); -/* label_start: */ - la_label(label_start); - la_andi(temp, src_temp, 1); - la_add_d(count, count, temp); - la_srli_d(src_temp, src_temp, 1); - la_addi_d(pop_count_ir2_opnd, pop_count_ir2_opnd, -1); - la_bne(pop_count_ir2_opnd, zero_ir2_opnd, label_start); + /* The zero-extended source occupies one independent 64-bit element. */ + la_movgr2fr_d(temp, src_opnd); + la_vpcnt_d(temp, temp); + la_movfr2gr_d(count, temp); ra_free_temp(temp); - /* 5. finish all steps, jump to exit */ - la_b(label_exit); -/* label_zero: */ - la_label(label_zero); + /* set O S Z A C P = 0 */ + la_x86mtflag(zero_ir2_opnd, 0x3f); + la_bne(src_opnd, zero_ir2_opnd, label_exit); /* set zf = 1 */ IR2_OPND n4095_opnd = ra_alloc_num_4095(); la_x86mtflag(n4095_opnd, 0x8); diff --git a/target/i386/latx/translator/tr-pattern.c b/target/i386/latx/translator/tr-pattern.c index ee15720703..9ddcc01113 100644 --- a/target/i386/latx/translator/tr-pattern.c +++ b/target/i386/latx/translator/tr-pattern.c @@ -12,6 +12,7 @@ #include "insts-pattern.h" #include "tu.h" #include "hbr.h" +#include "latx-unwind.h" #ifdef CONFIG_LATX_INSTS_PATTERN @@ -1394,6 +1395,69 @@ static bool translate_test_xxcc(IR1_INST *pir1) return true; } +#ifdef CONFIG_LATX_XCOMISX_OPT +static uint8_t ucomisd_seta_live_flags(IR1_INST *next) +{ +#ifdef CONFIG_LATX_FLAG_REDUCTION + if (option_flag_reduction && + ir1_opnd_is_gpr(ir1_get_opnd(next, 0))) { + TranslationBlock *tb = lsenv->tr_data->curr_tb; + uint8_t flags = 0; + + /* Keep all flags before a possible fault or an unknown successor. + * Only discard them across non-faulting register operations ending + * in an unconditional flag overwrite within this TB. */ + for (IR1_INST *inst = next + 1; inst <= tb_ir1_inst_last(tb); inst++) { + if (ir1_is_prefix_lock(inst)) { + return 0x3f; + } + for (int i = 0; i < ir1_opnd_num(inst); i++) { + IR1_OPND *opnd = ir1_get_opnd(inst, i); + if (!ir1_opnd_is_gpr(opnd) && !ir1_opnd_is_imm(opnd)) { + return 0x3f; + } + } + switch (ir1_opcode(inst)) { + case WRAP(ADD): + case WRAP(SUB): + case WRAP(CMP): + /* Require a retained, unfused write of every flag. */ + return inst->instptn.opc == INSTPTN_OPC_NONE && + ir1_get_eflag_def(inst) == 0x3f ? flags : 0x3f; + case WRAP(MOV): + case WRAP(MOVZX): + case WRAP(MOVSX): + case WRAP(MOVSXD): + case WRAP(NOP): + case WRAP(LAHF): + case WRAP(SETO): + case WRAP(SETNO): + case WRAP(SETB): + case WRAP(SETAE): + case WRAP(SETE): + case WRAP(SETNE): + case WRAP(SETBE): + case WRAP(SETA): + case WRAP(SETS): + case WRAP(SETNS): + case WRAP(SETP): + case WRAP(SETNP): + case WRAP(SETL): + case WRAP(SETGE): + case WRAP(SETLE): + case WRAP(SETG): + flags |= ir1_get_eflag_use(inst); + break; + default: + return 0x3f; + } + } + } +#endif + return 0x3f; +} +#endif + static bool translate_ucomisd_seta(IR1_INST *pir1) { IR1_INST *curr = pir1; @@ -1403,6 +1467,15 @@ static bool translate_ucomisd_seta(IR1_INST *pir1) IR1_OPND *opnd1 = ir1_get_opnd(curr, 1); IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src = load_freg128_from_ir1(opnd1); +#ifdef CONFIG_LATX_XCOMISX_OPT + uint8_t eflags = ucomisd_seta_live_flags(next); + const uint8_t seta_flags = CF_USEDEF_BIT | ZF_USEDEF_BIT; + /* Two comparisons can supply CF, ZF, PF and SETA together. */ + if ((eflags & PF_USEDEF_BIT) && (eflags & seta_flags)) { + eflags |= seta_flags; + } + IR2_OPND flag = ra_alloc_itemp(); +#else /* 0. set flag = 0 */ IR2_OPND flag_zf = ra_alloc_itemp(); IR2_OPND flag_pf = ra_alloc_itemp(); @@ -1427,19 +1500,25 @@ static bool translate_ucomisd_seta(IR1_INST *pir1) ra_free_temp(flag_zf); /* 4. mov flag to EFLAGS */ la_x86mtflag(flag, 0x3f); +#endif - lsenv->tr_data->curr_ir1_inst = next; - lsenv->tr_data->curr_ir1_count++; - switch (ir1_opcode(next)) { - case WRAP(SETA): +#ifdef CONFIG_LATX_XCOMISX_OPT + if ((eflags & seta_flags) == seta_flags) { + generate_xcomisx(dest, src, true, false, eflags); + la_setx86j(flag, COND_A); + } else +#endif + { + lsassert(ir1_opcode(next) == WRAP(SETA)); la_fcmp_cond_d(fcc0_ir2_opnd, src, dest, FCMP_COND_CLT); - break; - default: - lsassert(0); - break; + la_movcf2gr(flag, fcc0_ir2_opnd); +#ifdef CONFIG_LATX_XCOMISX_OPT + /* The compare can trap before even OF/AF/SF may be cleared. */ + generate_xcomisx(dest, src, true, false, eflags); +#endif } - - la_movcf2gr(flag, fcc0_ir2_opnd); + lsenv->tr_data->curr_ir1_inst = next; + lsenv->tr_data->curr_ir1_count++; store_ireg_to_ir1(flag, ir1_get_opnd(next, 0), false); ra_free_temp(flag); @@ -2379,6 +2458,94 @@ static bool translate_ucomiss_xx_jcc(IR1_INST *pir1) #endif +#ifdef CONFIG_LATX_AVX_OPT +static bool translate_avx_test_jcc(IR1_INST *pir1) +{ + IR1_INST *next = pir1->instptn.next; +#ifdef CONFIG_LATX_TU + TranslationBlock *tb = lsenv->tr_data->curr_tb; + if (tb->s_data->next_tb[TU_TB_INDEX_NEXT] && + tb->s_data->next_tb[TU_TB_INDEX_TARGET]) { + switch (ir1_opcode(pir1)) { + case WRAP(VPTEST): + translate_vptest(pir1); + break; + case WRAP(VTESTPS): + translate_vtestps(pir1); + break; + case WRAP(VTESTPD): + translate_vtestpd(pir1); + break; + default: + lsassert(0); + } + return translate_jcc(next); + } +#endif + IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0); + IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); + bool wide = ir1_opnd_is_ymm(opnd0); + IR2_OPND dest = wide ? load_freg256_from_ir1(opnd0) : + load_freg128_from_ir1(opnd0); + IR2_OPND src = wide ? load_freg256_from_ir1(opnd1) : + load_freg128_from_ir1(opnd1); + IR2_OPND flags = ra_alloc_itemp(); + IR2_OPND cond = ra_alloc_itemp(); + IR2_OPND target = ra_alloc_label(); + uint8_t eflags = ir1_get_eflag_def(pir1) & + ~lsenv->tr_data->curr_ir1_skipped_eflags; + int mask = ZF_BIT; + bool nonzero = true; + switch (ir1_opcode(next)) { + case WRAP(JNE): + nonzero = false; + break; + case WRAP(JAE): + nonzero = false; + /* fall through */ + case WRAP(JB): + mask = CF_BIT; + break; + case WRAP(JA): + nonzero = false; + /* fall through */ + case WRAP(JBE): + mask = ZF_BIT | CF_BIT; + break; + case WRAP(JE): + break; + default: + lsassert(0); + } + generate_avx_test_flags(pir1, dest, src, flags, + eflags | ir1_get_eflag_use(next)); + if (eflags) { + la_x86mtflag(flags, eflags); + } + if (mask == (ZF_BIT | CF_BIT)) { + la_andi(cond, flags, mask); + if (nonzero) { + la_bne(cond, zero_ir2_opnd, target); + } else { + la_beq(cond, zero_ir2_opnd, target); + } + } else { + IR2_OPND condition = mask == ZF_BIT ? fcc0_ir2_opnd : fcc1_ir2_opnd; + if (nonzero) { + la_bcnez(condition, target); + } else { + la_bceqz(condition, target); + } + } + ra_free_temp(cond); + ra_free_temp(flags); + tr_generate_exit_tb(next, 0); + la_label(target); + tr_generate_exit_tb(next, 1); + return true; +} +#endif + bool try_translate_instptn(IR1_INST *pir1) { instptn_check_false(); @@ -2392,6 +2559,13 @@ bool try_translate_instptn(IR1_INST *pir1) case INSTPTN_OPC_CMP_JCC: return translate_cmp_jcc(pir1); case INSTPTN_OPC_TEST_JCC: +#ifdef CONFIG_LATX_AVX_OPT + if (ir1_opcode(pir1) == WRAP(VPTEST) || + ir1_opcode(pir1) == WRAP(VTESTPS) || + ir1_opcode(pir1) == WRAP(VTESTPD)) { + return translate_avx_test_jcc(pir1); + } +#endif return translate_test_jcc(pir1); case INSTPTN_OPC_CQO_IDIV: return translate_cqo_idiv(pir1); @@ -2467,195 +2641,206 @@ bool try_translate_instptn(IR1_INST *pir1) return false; } -void opt_instptn_fix(CPUState *cpu, TranslationBlock *tb, int index) +unsigned instptn_flags_recovery(IR1_INST *pir1) { - CPUArchState *env = cpu->env_ptr; - // int num_insns = tb->icount; - for (int i = 0; i < index; i++) { - IR1_INST *pir1 = tb_ir1_inst(tb, i); - if (pir1->instptn.opc == INSTPTN_OPC_CMP_XX_JCC || - pir1->instptn.opc == INSTPTN_OPC_TEST_XX_JCC || - pir1->instptn.opc == INSTPTN_OPC_BT_XX_JCC) { - - int opnd_size = ir1_opnd_size(ir1_get_opnd(pir1, 0)); - ucontext_t *uc = env->puc; - - switch (ir1_opcode(pir1)) - { - case WRAP(CMP): - { - uint64_t a0 = UC_GR(uc)[4]; - uint64_t a1 = UC_GR(uc)[5]; - uint64_t eflags = 0; - switch (opnd_size) - { - case 8: - asm volatile ( - DEFINE_PARSE_R - "parse_r_%= __src, %[src] \n\t" - "parse_r_%= __dst, %[dst] \n\t" - ".word (0x003f0008 | (__src << 10) | (__dst << 5))\n\t" /*x86sub.b*/ - "parse_r_%= __flags, %[flags] \n\t" - ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ - : [dst]"+r"(a0), - [flags]"=r"(eflags) - : [src]"r"(a1) - : "cc" - ); - break; - case 16: - asm volatile ( - DEFINE_PARSE_R - "parse_r_%= __src, %[src] \n\t" - "parse_r_%= __dst, %[dst] \n\t" - ".word (0x003f0009 | (__src << 10) | (__dst << 5))\n\t" /*x86sub.h*/ - "parse_r_%= __flags, %[flags] \n\t" - ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ - : [dst]"+r"(a0), - [flags]"=r"(eflags) - : [src]"r"(a1) - : "cc" - ); - break; - case 32: - asm volatile ( - DEFINE_PARSE_R - "parse_r_%= __src, %[src] \n\t" - "parse_r_%= __dst, %[dst] \n\t" - ".word (0x003f000a | (__src << 10) | (__dst << 5))\n\t" /*x86sub.w*/ - "parse_r_%= __flags, %[flags] \n\t" - ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ - : [dst]"+r"(a0), - [flags]"=r"(eflags) - : [src]"r"(a1) - : "cc" - ); - break; - case 64: - asm volatile ( - DEFINE_PARSE_R - "parse_r_%= __src, %[src] \n\t" - "parse_r_%= __dst, %[dst] \n\t" - ".word (0x003f000b | (__src << 10) | (__dst << 5))\n\t" /*x86sub.d*/ - "parse_r_%= __flags, %[flags] \n\t" - ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ - : [dst]"+r"(a0), - [flags]"=r"(eflags) - : [src]"r"(a1) - : "cc" - ); - break; - default: - break; - } - env->eflags = env->eflags & ~(0b100011010101); - env->eflags = env->eflags | (eflags & 0b100011010101); - } - break; - case WRAP(TEST): - { - IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0); - IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); - - bool is_same_reg = ir1_opnd_is_same_reg(opnd0, opnd1); - uint64_t a0 = UC_GR(uc)[4]; - uint64_t a1 = is_same_reg ? a0 : UC_GR(uc)[5]; - uint64_t eflags = 0; - - switch (opnd_size) - { - case 8: - asm volatile ( - DEFINE_PARSE_R - "parse_r_%= __src, %[src] \n\t" - "parse_r_%= __dst, %[dst] \n\t" - ".word (0x003f8010 | (__src << 10) | (__dst << 5))\n\t" /*x86and.b*/ - "parse_r_%= __flags, %[flags] \n\t" - ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ - : [dst]"+r"(a0), - [flags]"=r"(eflags) - : [src]"r"(a1) - : "cc" - ); - break; - case 16: - asm volatile ( - DEFINE_PARSE_R - "parse_r_%= __src, %[src] \n\t" - "parse_r_%= __dst, %[dst] \n\t" - ".word (0x003f8011 | (__src << 10) | (__dst << 5))\n\t" /*x86and.h*/ - "parse_r_%= __flags, %[flags] \n\t" - ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ - : [dst]"+r"(a0), - [flags]"=r"(eflags) - : [src]"r"(a1) - : "cc" - ); - break; - case 32: - asm volatile ( - DEFINE_PARSE_R - "parse_r_%= __src, %[src] \n\t" - "parse_r_%= __dst, %[dst] \n\t" - ".word (0x003f8012 | (__src << 10) | (__dst << 5))\n\t" /*x86and.w*/ - "parse_r_%= __flags, %[flags] \n\t" - ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ - : [dst]"+r"(a0), - [flags]"=r"(eflags) - : [src]"r"(a1) - : "cc" - ); - break; - case 64: - asm volatile ( - DEFINE_PARSE_R - "parse_r_%= __src, %[src] \n\t" - "parse_r_%= __dst, %[dst] \n\t" - ".word (0x003f8013 | (__src << 10) | (__dst << 5))\n\t" /*x86and.d*/ - "parse_r_%= __flags, %[flags] \n\t" - ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ - : [dst]"+r"(a0), - [flags]"=r"(eflags) - : [src]"r"(a1) - : "cc" - ); - break; - default: - break; - } - env->eflags = env->eflags & ~(0b100011000101); - env->eflags = env->eflags | (eflags & 0b100011000101); - } - break; - case WRAP(BT): - { - uint64_t a0 = UC_GR(uc)[4]; - uint64_t a1 = UC_GR(uc)[5]; - env->eflags = env->eflags & ~(1); // clear CF - env->eflags = env->eflags | ((a0 >> a1) & 1); // set CF - } + unsigned recovery; + + switch (pir1->instptn.opc) { + case INSTPTN_OPC_CMP_XX_JCC: + if (ir1_opcode(pir1) != WRAP(CMP)) { + return INSTPTN_FLAGS_NONE; + } + recovery = INSTPTN_FLAGS_CMP; + break; + case INSTPTN_OPC_TEST_XX_JCC: + if (ir1_opcode(pir1) != WRAP(TEST)) { + return INSTPTN_FLAGS_NONE; + } + recovery = INSTPTN_FLAGS_TEST; + if (ir1_opnd_is_same_reg(ir1_get_opnd(pir1, 0), + ir1_get_opnd(pir1, 1))) { + recovery |= INSTPTN_FLAGS_SAME_REG; + } + break; + case INSTPTN_OPC_BT_XX_JCC: + if (ir1_opcode(pir1) != WRAP(BT)) { + return INSTPTN_FLAGS_NONE; + } + recovery = INSTPTN_FLAGS_BT; + break; + default: + return INSTPTN_FLAGS_NONE; + } + + int size = ir1_opnd_size(ir1_get_opnd(pir1, 0)); + + lsassert(size == 8 || size == 16 || size == 32 || size == 64); + return recovery | ((__builtin_ctz(size) - 3) << INSTPTN_FLAGS_SIZE_SHIFT); +} + +void opt_instptn_fix(CPUArchState *env, unsigned recovery) +{ + int opnd_size = 8 << ((recovery >> INSTPTN_FLAGS_SIZE_SHIFT) & + INSTPTN_FLAGS_SIZE_MASK); + ucontext_t *uc = env->puc; + + switch (recovery & INSTPTN_FLAGS_OP_MASK) + { + case INSTPTN_FLAGS_CMP: + { + uint64_t a0 = UC_GR(uc)[4]; + uint64_t a1 = UC_GR(uc)[5]; + uint64_t eflags = 0; + switch (opnd_size) + { + case 8: + asm volatile ( + DEFINE_PARSE_R + "parse_r_%= __src, %[src] \n\t" + "parse_r_%= __dst, %[dst] \n\t" + ".word (0x003f0008 | (__src << 10) | (__dst << 5))\n\t" /*x86sub.b*/ + "parse_r_%= __flags, %[flags] \n\t" + ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ + : [dst]"+r"(a0), + [flags]"=r"(eflags) + : [src]"r"(a1) + : "cc" + ); + break; + case 16: + asm volatile ( + DEFINE_PARSE_R + "parse_r_%= __src, %[src] \n\t" + "parse_r_%= __dst, %[dst] \n\t" + ".word (0x003f0009 | (__src << 10) | (__dst << 5))\n\t" /*x86sub.h*/ + "parse_r_%= __flags, %[flags] \n\t" + ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ + : [dst]"+r"(a0), + [flags]"=r"(eflags) + : [src]"r"(a1) + : "cc" + ); + break; + case 32: + asm volatile ( + DEFINE_PARSE_R + "parse_r_%= __src, %[src] \n\t" + "parse_r_%= __dst, %[dst] \n\t" + ".word (0x003f000a | (__src << 10) | (__dst << 5))\n\t" /*x86sub.w*/ + "parse_r_%= __flags, %[flags] \n\t" + ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ + : [dst]"+r"(a0), + [flags]"=r"(eflags) + : [src]"r"(a1) + : "cc" + ); + break; + case 64: + asm volatile ( + DEFINE_PARSE_R + "parse_r_%= __src, %[src] \n\t" + "parse_r_%= __dst, %[dst] \n\t" + ".word (0x003f000b | (__src << 10) | (__dst << 5))\n\t" /*x86sub.d*/ + "parse_r_%= __flags, %[flags] \n\t" + ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ + : [dst]"+r"(a0), + [flags]"=r"(eflags) + : [src]"r"(a1) + : "cc" + ); + break; + default: break; - case WRAP(JB): - case WRAP(JAE): - case WRAP(JE): - case WRAP(JNE): - case WRAP(JBE): - case WRAP(JA): - case WRAP(JL): - case WRAP(JGE): - case WRAP(JLE): - case WRAP(JG): - case WRAP(JS): - case WRAP(JNS): - case WRAP(JNO): - case WRAP(JO): + } + env->eflags = env->eflags & ~(0b100011010101); + env->eflags = env->eflags | (eflags & 0b100011010101); + } + break; + case INSTPTN_FLAGS_TEST: + { + bool is_same_reg = recovery & INSTPTN_FLAGS_SAME_REG; + uint64_t a0 = UC_GR(uc)[4]; + uint64_t a1 = is_same_reg ? a0 : UC_GR(uc)[5]; + uint64_t eflags = 0; + + switch (opnd_size) + { + case 8: + asm volatile ( + DEFINE_PARSE_R + "parse_r_%= __src, %[src] \n\t" + "parse_r_%= __dst, %[dst] \n\t" + ".word (0x003f8010 | (__src << 10) | (__dst << 5))\n\t" /*x86and.b*/ + "parse_r_%= __flags, %[flags] \n\t" + ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ + : [dst]"+r"(a0), + [flags]"=r"(eflags) + : [src]"r"(a1) + : "cc" + ); + break; + case 16: + asm volatile ( + DEFINE_PARSE_R + "parse_r_%= __src, %[src] \n\t" + "parse_r_%= __dst, %[dst] \n\t" + ".word (0x003f8011 | (__src << 10) | (__dst << 5))\n\t" /*x86and.h*/ + "parse_r_%= __flags, %[flags] \n\t" + ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ + : [dst]"+r"(a0), + [flags]"=r"(eflags) + : [src]"r"(a1) + : "cc" + ); + break; + case 32: + asm volatile ( + DEFINE_PARSE_R + "parse_r_%= __src, %[src] \n\t" + "parse_r_%= __dst, %[dst] \n\t" + ".word (0x003f8012 | (__src << 10) | (__dst << 5))\n\t" /*x86and.w*/ + "parse_r_%= __flags, %[flags] \n\t" + ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ + : [dst]"+r"(a0), + [flags]"=r"(eflags) + : [src]"r"(a1) + : "cc" + ); + break; + case 64: + asm volatile ( + DEFINE_PARSE_R + "parse_r_%= __src, %[src] \n\t" + "parse_r_%= __dst, %[dst] \n\t" + ".word (0x003f8013 | (__src << 10) | (__dst << 5))\n\t" /*x86and.d*/ + "parse_r_%= __flags, %[flags] \n\t" + ".word (0x005c0000 | (0x3f << 10) | __flags)\n\t" /*x86mfflag*/ + : [dst]"+r"(a0), + [flags]"=r"(eflags) + : [src]"r"(a1) + : "cc" + ); + break; + default: break; - default: - lsassert(0); - break; - } - return; } + env->eflags = env->eflags & ~(0b100011000101); + env->eflags = env->eflags | (eflags & 0b100011000101); + } + break; + case INSTPTN_FLAGS_BT: + { + uint64_t a0 = UC_GR(uc)[4]; + uint64_t a1 = UC_GR(uc)[5]; + env->eflags = env->eflags & ~(1); // clear CF + env->eflags = env->eflags | ((a0 >> a1) & 1); // set CF + } + break; + case INSTPTN_FLAGS_NONE: + break; + default: + lsassert(0); + break; } } #undef WRAP diff --git a/target/i386/latx/translator/translate.c b/target/i386/latx/translator/translate.c index 7d6b364694..5b0fde9538 100644 --- a/target/i386/latx/translator/translate.c +++ b/target/i386/latx/translator/translate.c @@ -20,6 +20,7 @@ #include "latx-smc.h" #include "jrra.h" #include "latx-native-asm.h" +#include "latx-unwind.h" extern void *helper_tb_lookup_ptr(CPUArchState *); static int ss_generate_match_fail_native_code(void* code_buf); @@ -1924,6 +1925,13 @@ bool ir1_translate(IR1_INST *ir1) IR1_OPND *opnd = ir1_get_opnd(ir1, i); if (ir1_opnd_is_ymm(opnd)) { + /* A high insert reads only src1's low 128 bits. */ + if (i == 1 && + (ir1_opcode(ir1) == dt_X86_INS_VINSERTF128 || + ir1_opcode(ir1) == dt_X86_INS_VINSERTI128) && + (ir1_opnd_uimm(ir1_get_opnd(ir1, 3)) & 1)) { + continue; + } flush_pending_ymmh_zero( ra_alloc_xmm(ir1_opnd_base_reg_num(opnd))); } else if (ir1_opnd_is_mem(opnd) && @@ -2497,6 +2505,11 @@ int tr_ir2_generate(struct TranslationBlock *tb) bool reduce_proepo = false; bool x87_fcsr_active = false; int softfpu_region_end = -1; +#ifdef CONFIG_LATX_INSTS_PATTERN + unsigned flags_recovery = INSTPTN_FLAGS_NONE; +#endif + + QEMU_BUILD_BUG_ON(CC_OP_NB > LATX_UNWIND_CC_OP_MASK + 1); lsenv->tr_data->softfpu_region_active = false; @@ -2586,11 +2599,14 @@ int tr_ir2_generate(struct TranslationBlock *tb) * at the end of TB translate cache. */ tcg_ctx->gen_insn_data[i][0] = pir1->info->address; - /* Preserve logical upper zeros if this instruction faults before - * the deferred clears at the normal TB exit have executed. The - * low 16 bits remain the condition-code operation. */ + /* Persist recovery state with the code, including loaded AOT TBs. */ tcg_ctx->gen_insn_data[i][1] = - (target_ulong)lsenv->tr_data->ymmh_zero_pending << 16; + (target_ulong)lsenv->tr_data->ymmh_zero_pending << + LATX_UNWIND_YMMH_SHIFT; +#ifdef CONFIG_LATX_INSTS_PATTERN + tcg_ctx->gen_insn_data[i][1] |= + flags_recovery << LATX_UNWIND_INSTPTN_SHIFT; +#endif #ifdef CONFIG_LATX_IMM_REG imm_cache->curr_ir1_index = i; @@ -2630,6 +2646,16 @@ int tr_ir2_generate(struct TranslationBlock *tb) } bool translation_success = ir1_translate(pir1); +#ifdef CONFIG_LATX_INSTS_PATTERN + if (translation_success) { + unsigned recovery = instptn_flags_recovery(pir1); + + /* A fault in the comparison itself still needs the old flags. */ + if (recovery != INSTPTN_FLAGS_NONE) { + flags_recovery = recovery; + } + } +#endif if (!translation_success) { #ifdef CONFIG_LATX_TU tb->s_data->tu_tb_mode = BAD_TB; diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index cfb1376b29..1fb3cb2dfb 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -37,6 +37,10 @@ #ifdef CONFIG_LATX #include "reg-map.h" #include "latx-options.h" +#include "latx-unwind.h" +#ifdef CONFIG_LATX_INSTS_PATTERN +#include "insts-pattern.h" +#endif #endif #define PREFIX_REPZ 0x01 @@ -8752,8 +8756,8 @@ static void restore_extcontext(CPUX86State *env, ucontext_t *uc) void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb, target_ulong *data) { - int cc_op = data[1] & 0xffff; - uint16_t ymmh_zero_pending = data[1] >> 16; + int cc_op = data[1] & LATX_UNWIND_CC_OP_MASK; + uint16_t ymmh_zero_pending = data[1] >> LATX_UNWIND_YMMH_SHIFT; /* env->eip = data[0] - tb->cs_base; */ env->eip = data[0]; @@ -8788,6 +8792,10 @@ void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb, env->regs[R_R15] = UC_GR(uc)[reg_gpr_map[r15_index]]; #endif restore_extcontext(env, uc); +#ifdef CONFIG_LATX_INSTS_PATTERN + opt_instptn_fix(env, (data[1] >> LATX_UNWIND_INSTPTN_SHIFT) & + LATX_UNWIND_INSTPTN_MASK); +#endif /* The saved host registers can still contain stale upper halves for * completed VEX.128 instructions whose clears were deferred. */ if (option_enable_lasx) { diff --git a/tests/integration/avx-scalar-double-alias.asm b/tests/integration/avx-scalar-double-alias.asm new file mode 100644 index 0000000000..8c149fe5c6 --- /dev/null +++ b/tests/integration/avx-scalar-double-alias.asm @@ -0,0 +1,82 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +section .text + +%macro CHECK_RESULT 2 + mov edi, %2 + vmovdqu [actual], ymm0 + mov rax, [actual] + cmp rax, [%1] + jne fail + mov rax, [actual + 8] + cmp rax, [input_a + 8] + jne fail + mov rax, [actual + 16] + or rax, [actual + 24] + jne fail +%endmacro + +%macro CHECK_ALIASES 3 + vmovdqu ymm0, [junk] + vmovdqu ymm1, [input_a] + vmovdqu ymm2, [input_b] + %1 xmm0, xmm1, xmm2 + CHECK_RESULT %2, 1 + + vmovdqu ymm0, [input_a] + %1 xmm0, xmm0, xmm2 + CHECK_RESULT %2, 2 + + vmovdqu ymm0, [input_b] + %1 xmm0, xmm1, xmm0 + CHECK_RESULT %2, 3 + + vmovdqu ymm0, [junk] + %1 xmm0, xmm1, xmm1 + CHECK_RESULT %3, 4 + + vmovdqu ymm0, [input_a] + %1 xmm0, xmm0, xmm0 + CHECK_RESULT %3, 5 + + vmovdqu ymm0, [junk] + %1 xmm0, xmm1, [input_b] + CHECK_RESULT %2, 6 + + vmovdqu ymm0, [input_a] + %1 xmm0, xmm0, [input_b] + CHECK_RESULT %2, 7 +%endmacro + +_start: + mov edi, 1 + CHECK_ALIASES vaddsd, sum_ab, sum_aa + CHECK_ALIASES vsubsd, diff_ab, diff_aa + CHECK_ALIASES vmulsd, product_ab, product_aa + CHECK_ALIASES vdivsd, quotient_ab, quotient_aa + xor edi, edi +fail: + mov eax, 60 + syscall + +section .rodata align=32 +input_a: dq 6.0, 0x1122334455667788, -1, -1 +input_b: dq 2.0, 0x8877665544332211, -1, -1 +junk: times 4 dq 0xdeadbeefdeadbeef +sum_ab: dq 8.0 +sum_aa: dq 12.0 +diff_ab: dq 4.0 +diff_aa: dq 0.0 +product_ab: dq 12.0 +product_aa: dq 36.0 +quotient_ab: dq 3.0 +quotient_aa: dq 1.0 + +section .bss align=32 +actual: resb 32 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/avx-single-instruction-boundaries.asm b/tests/integration/avx-single-instruction-boundaries.asm index 759440e374..b137469bae 100644 --- a/tests/integration/avx-single-instruction-boundaries.asm +++ b/tests/integration/avx-single-instruction-boundaries.asm @@ -45,6 +45,39 @@ section .text %endif %endmacro +%macro CHECK_SCALAR_COPY 1 + vmovdqu ymm0, [junk] + vmovdqu ymm1, [scalar_input] + %1 xmm0, xmm1, xmm1 + vmovdqu [scalar_reference], ymm0 + vmovdqu ymm1, [scalar_input] + %1 xmm1, xmm1, xmm1 + vmovdqu ymm0, ymm1 + CHECK scalar_reference, 14 + vmovdqu ymm1, [scalar_input] + %1 xmm1, xmm1, [scalar_input] + vmovdqu ymm0, ymm1 + CHECK scalar_reference, 15 +%endmacro + +%macro BLEND 2 + vmovdqu ymm0, [junk] + vmovdqu ymm1, [words2] + vmovdqu ymm2, [words3] + %if %2 = 1 + vmovdqu ymm0, ymm1 + vpblendd xmm0, xmm0, xmm2, %1 + %elif %2 = 2 + vmovdqu ymm0, ymm2 + vpblendd xmm0, xmm1, xmm0, %1 + %elif %2 = 3 + vpblendd xmm0, xmm1, [words3], %1 + %else + vpblendd xmm0, xmm1, xmm2, %1 + %endif + CHECK blend_ %+ %1, 16 +%endmacro + _start: %assign alias 0 %rep 3 @@ -52,7 +85,38 @@ _start: MADD vpmaddwd, alias, words2, words3, maddwd, ymm MADD vpmaddubsw, alias, bytes255, bytes128, maddubsw, xmm MADD vpmaddubsw, alias, bytes255, bytes128, maddubsw, ymm + MADD vpandn, alias, words2, words3, andnot, xmm + MADD vpandn, alias, words2, words3, andnot, ymm %assign alias alias + 1 +%endrep + + vmovdqu ymm0, [junk] + vpandn xmm0, xmm0, xmm0 + CHECK zeros, 9 + vmovdqu ymm1, [words2] + vpandn xmm0, xmm1, [words3] + CHECK andnot_xmm, 10 + + vmovdqu ymm0, [junk] + vpbroadcastq ymm0, xmm0 + CHECK junk, 11 + vmovdqu ymm1, [broadcast_input] + vpbroadcastq ymm0, xmm1 + CHECK broadcast_q, 12 + vpbroadcastq ymm0, [broadcast_input] + CHECK broadcast_q, 13 + + CHECK_SCALAR_COPY vrcpss + CHECK_SCALAR_COPY vrsqrtss + +%assign blend_mask 0 +%rep 16 +%assign blend_alias 0 +%rep 4 + BLEND blend_mask, blend_alias +%assign blend_alias blend_alias + 1 +%endrep +%assign blend_mask blend_mask + 1 %endrep vmovdqu ymm1, [words2] @@ -94,6 +158,9 @@ maddwd_xmm: times 4 dd 12 maddubsw: times 16 dw 0x8000 maddubsw_xmm: times 8 dw 0x8000 times 8 dw 0 +andnot: times 16 dw 1 +andnot_xmm: times 8 dw 1 + times 8 dw 0 zeros: times 4 dq 0 ones16: times 16 dw 1 ones32: times 8 dd 1 @@ -102,8 +169,28 @@ count15: dq 15, 0 count31: dq 31, 0 count63: dq 63, 0 count64: dq 64, 0 +broadcast_input: dq 0x0123456789abcdef, 0xfedcba9876543210, 0, -1 +broadcast_q: times 4 dq 0x0123456789abcdef +scalar_input: dd 0x40800000, 0x11223344, 0x55667788, 0x99aabbcc + times 4 dd 0xdeadbeef +%assign blend_mask 0 +%rep 16 +blend_ %+ blend_mask: +%assign lane 0 +%rep 4 + %if blend_mask & (1 << lane) + dd 0x00030003 + %else + dd 0x00020002 + %endif +%assign lane lane + 1 +%endrep + times 4 dd 0 +%assign blend_mask blend_mask + 1 +%endrep section .bss align=32 actual: resb 32 +scalar_reference: resb 32 section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/avx-test-flags.asm b/tests/integration/avx-test-flags.asm new file mode 100644 index 0000000000..b13f747a50 --- /dev/null +++ b/tests/integration/avx-test-flags.asm @@ -0,0 +1,89 @@ +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +%macro CHECK 6 + vmovdqu ymm0, [%3] + vmovdqu ymm1, [%4] + %1 %2 %+ 0, %2 %+ 1 +%if %6 & 1 + setc [result] +%endif +%if %6 & 2 + setp [result + 1] +%endif +%if %6 & 4 + lahf + mov [result + 2], ah +%endif +%if %6 & 8 + setz [result + 3] +%endif +%if %6 & 16 + sets [result + 4] +%endif +%if %6 & 32 + seto [result + 5] +%endif + xor eax, eax +%if %6 & 1 + cmp byte [result], (%5 & 1) != 0 + jne fail +%endif +%if %6 & 2 + cmp byte [result + 1], 0 + jne fail +%endif +%if %6 & 4 + cmp byte [result + 2], %5 | 2 + jne fail +%endif +%if %6 & 8 + cmp byte [result + 3], (%5 & 0x40) != 0 + jne fail +%endif +%if %6 & 16 + cmp byte [result + 4], 0 + jne fail +%endif +%if %6 & 32 + cmp byte [result + 5], 0 + jne fail +%endif +%endmacro + +%macro SUITE 2 +%assign mask 0 +%rep 64 + CHECK %1, %2, zeros, zeros, 0x41, mask + CHECK %1, %2, zeros, ones, 0x40, mask + CHECK %1, %2, ones, ones, 1, mask + CHECK %1, %2, half, ones, 0, mask +%assign mask mask + 1 +%endrep +%endmacro + +section .text +_start: + SUITE vptest, xmm + SUITE vptest, ymm + SUITE vtestps, xmm + SUITE vtestps, ymm + SUITE vtestpd, xmm + SUITE vtestpd, ymm + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +zeros: times 4 dq 0 +ones: times 4 dq -1 +half: dq -1, 0, -1, 0 +section .bss +result: resb 6 +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/avx-test-jcc.asm b/tests/integration/avx-test-jcc.asm new file mode 100644 index 0000000000..f3cb7a25d7 --- /dev/null +++ b/tests/integration/avx-test-jcc.asm @@ -0,0 +1,66 @@ +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +; Check both the branch outcome and flags visible in its successor. +%macro CHECK 7 + vmovdqu ymm0, [%3] + vmovdqu ymm1, [%4] + %1 %2 %+ 0, %2 %+ 1 + %5 %%taken + mov r9d, 0 + jmp %%flags +%%taken: + mov r9d, 1 +%%flags: + pushfq + pop rax + and eax, 0x8d5 + cmp eax, %6 + jne fail + cmp r9d, (%7) & 1 + jne fail +%endmacro + +%macro RELATION 5 + CHECK %1, %2, %3, %4, je, %5, (%5 & 0x40) != 0 + CHECK %1, %2, %3, %4, jne, %5, (%5 & 0x40) = 0 + CHECK %1, %2, %3, %4, jb, %5, (%5 & 1) != 0 + CHECK %1, %2, %3, %4, jae, %5, (%5 & 1) = 0 + CHECK %1, %2, %3, %4, jbe, %5, (%5 & 0x41) != 0 + CHECK %1, %2, %3, %4, ja, %5, (%5 & 0x41) = 0 +%endmacro + +%macro SUITE 2 + RELATION %1, %2, zeros, zeros, 0x41 + RELATION %1, %2, zeros, ones, 0x40 + RELATION %1, %2, ones, ones, 1 + RELATION %1, %2, half, ones, 0 +%endmacro + +section .text +_start: + mov ecx, 128 +again: + SUITE vptest, xmm + SUITE vptest, ymm + SUITE vtestps, xmm + SUITE vtestps, ymm + SUITE vtestpd, xmm + SUITE vtestpd, ymm + dec ecx + jnz again + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +zeros: times 4 dq 0 +ones: times 4 dq -1 +half: dq -1, 0, -1, 0 +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/base-displacement.asm b/tests/integration/base-displacement.asm new file mode 100644 index 0000000000..d0bcdd220a --- /dev/null +++ b/tests/integration/base-displacement.asm @@ -0,0 +1,260 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +%macro CHECK_OFFSET 1 + lea rbx, [data] + mov r14, rbx + mov rax, %1 + sub rbx, rax + mov rax, [rbx + %1] + cmp rax, [data] + jne fail + mov rax, 0x8877665544332211 + mov [rbx + %1], rax + cmp [data], rax + jne fail + lea rax, [rbx + %1] + cmp rax, r14 + jne fail + mov rax, [rbx + %1 + 8] + cmp rax, [data + 8] + jne fail + vmovdqu ymm0, [vector] + vmovdqu [rbx + %1], ymm0 + vmovdqu ymm1, [rbx + %1] + vpxor ymm1, ymm1, ymm0 + vptest ymm1, ymm1 + jnz fail + mov rax, [data] + mov rcx, 5 + add qword [rbx + %1], rcx + add rax, rcx + cmp [data], rax + jne fail + xchg rax, [rbx + %1] + cmp [data], rax + jne fail + mov ecx, 1 + bt qword [rbx + %1], rcx + setc dl + bt qword [data], rcx + setc cl + cmp dl, cl + jne fail +%endmacro + +section .text +_start: + CHECK_OFFSET -2147483648 + CHECK_OFFSET -65537 + CHECK_OFFSET -29904 + CHECK_OFFSET -29124 + CHECK_OFFSET -4097 + CHECK_OFFSET -2049 + CHECK_OFFSET -2048 + CHECK_OFFSET 0 + CHECK_OFFSET 2047 + CHECK_OFFSET 2048 + CHECK_OFFSET 4095 + CHECK_OFFSET 4096 + CHECK_OFFSET 2147481599 + CHECK_OFFSET 2147481600 + CHECK_OFFSET 2147483632 + + ; An aliased load must consume the base before publishing its result. + lea rax, [data - 29904] + mov rax, [rax + 29904] + cmp rax, [data] + jne fail + lea rax, [data + 29904] + mov eax, [rax - 29904] + cmp eax, [data] + jne fail + lea rbp, [data - 29904] + mov rbp, [rbp + 29904] + cmp rbp, [data] + jne fail + mov r15, rsp + lea rsp, [data + 29904] + mov rsp, [rsp - 29904] + mov rax, rsp + mov rsp, r15 + cmp rax, [data] + jne fail + lea rbx, [data - 29904] + mov [rbx + 29904], rbx + cmp [data], rbx + jne fail + + ; Register bit offsets may move the access away from the nominal base. + mov qword [data], 1 + mov qword [data + 8], 0 + mov ecx, 64 + bts qword [rbx + 29904], rcx + jc fail + cmp qword [data + 8], 1 + jne fail + mov rcx, -64 + btr qword [rbx + 29912], rcx + jnc fail + cmp qword [data], 0 + jne fail + + ; Atomic and no-offset callers retain complete addresses. + mov qword [data], 7 + mov eax, 7 + mov edx, 11 + lock cmpxchg [rbx + 29904], rdx + jne fail + cmp qword [data], 11 + jne fail + mov edx, 5 + lock xadd [rbx + 29904], rdx + cmp edx, 11 + jne fail + cmp qword [data], 16 + jne fail + + ; Legacy SSE shares the helper. x87 needs room for a second access at +8. + movdqu xmm0, [vector] + movdqu [rbx + 29904], xmm0 + movdqu xmm1, [rbx + 29904] + pxor xmm1, xmm0 + ptest xmm1, xmm1 + jnz fail + lea rbx, [data - 6143] + fld tword [one] + fstp tword [rbx + 6143] + fld tword [rbx + 6143] + fstp tword [scratch] + mov rax, [scratch] + cmp rax, [one] + jne fail + mov ax, [scratch + 8] + cmp ax, [one + 8] + jne fail + + ; Address-size, index and segment fallbacks must still include every term. + lea rbx, [data] + sub rbx, 29904 + mov rax, [ebx + 29904] + cmp rax, [data] + jne fail + mov ecx, 7 + sub rbx, rcx + mov rax, [rbx + rcx + 29904] + cmp rax, [data] + jne fail + mov eax, 158 + mov edi, 0x1002 + lea rsi, [data] + syscall + test eax, eax + jnz fail + mov rbx, 29904 + mov rax, [fs:rbx - 29904] + cmp rax, [data] + jne fail + mov eax, 158 + mov edi, 0x1002 + xor esi, esi + syscall + + ; Fault on each crossing access, reopen the protected region, then retry. + mov eax, 13 + mov edi, 11 + lea rsi, [action] + xor edx, edx + mov r10d, 8 + syscall + test eax, eax + jnz fail + mov eax, 9 + xor edi, edi + mov esi, 196608 + mov edx, 3 + mov r10d, 0x22 + mov r8, -1 + xor r9d, r9d + syscall + cmp rax, -4095 + jae fail + mov r12, rax + lea r13, [r12 + 65536] + lea rbx, [r13 - 16 + 29904] + vmovdqu ymm0, [vector] + vmovdqu [rbx - 29904], ymm0 + call protect + lea rax, [fault_load] + mov [expected_pc], rax +fault_load: + vmovdqu ymm1, [rbx - 29904] + vpxor ymm2, ymm1, ymm0 + vptest ymm2, ymm2 + jnz fail + call protect + lea rax, [fault_store] + mov [expected_pc], rax +fault_store: + vmovdqu [rbx - 29904], ymm0 + cmp dword [faults], 2 + jne fail + vmovdqu ymm2, [rbx - 29904] + vpxor ymm2, ymm2, ymm0 + vptest ymm2, ymm2 + jnz fail + xor edi, edi + jmp exit +protect: + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + xor edx, edx + syscall + test eax, eax + jnz fail + ret +handler: + cmp edi, 11 + jne fail + cmp [rsi + 16], r13 + jne fail + mov rax, [expected_pc] + cmp [rdx + 168], rax + jne fail + inc dword [faults] + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + mov edx, 3 + syscall + test eax, eax + jnz fail + ret +restorer: + mov eax, 15 + syscall + ud2 +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +vector: dq 0x8877665544332211, 0x1122334455667788 + dq 0x0123456789abcdef, 0xfedcba9876543210 +one: dq 0x8000000000000000 + dw 0x3fff +action: dq handler, 0x04000004, restorer, 0 +section .data align=32 +data: times 4 dq 0x8877665544332211 +section .bss align=8 +faults: resd 1 +alignb 8 +expected_pc: resq 1 +scratch: resb 16 +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/instptn-signal-recovery-i386.S b/tests/integration/instptn-signal-recovery-i386.S new file mode 100644 index 0000000000..e81c64eb54 --- /dev/null +++ b/tests/integration/instptn-signal-recovery-i386.S @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +.macro compare kind, bits, same=0 +.if \bits == 8 +.if \same + \kind %al, %al +.else + \kind %bl, %al +.endif +.elseif \bits == 16 +.if \same + \kind %ax, %ax +.else + \kind %bx, %ax +.endif +.else +.if \same + \kind %eax, %eax +.else + \kind %ebx, %eax +.endif +.endif +.endm + +.macro run_case name, kind, bits, mask, same=0 + mov $0x80008080, %eax + mov $0x7fff7f81, %ebx +.ifc \kind,bt + compare bt, \bits +.else + compare \kind, \bits, \same +.endif + pushfl + popl expected_flags + movl $\mask, flag_mask + movl $\name\()_fault, expected_pc + call arm_fault + call \name + call check_result +.endm + +.macro fault_function name, kind, bits, same=0 +.p2align 4 +\name: +.ifc \kind,bt + compare bt, \bits +.else + compare \kind, \bits, \same +.endif +\name\()_fault: + mov (%edi), %ecx +.ifc \kind,bt + jc 1f +.else + jz 1f +.endif + ret +1: + ret +.endm + +.text +.globl _start +_start: + mov $174, %eax + mov $11, %ebx + mov $action, %ecx + xor %edx, %edx + mov $8, %esi + int $0x80 + test %eax, %eax + js fail_setup + mov $192, %eax + xor %ebx, %ebx + mov $16384, %ecx + mov $3, %edx + mov $0x21, %esi + mov $-1, %edi + xor %ebp, %ebp + int $0x80 + cmp $-4095, %eax + jae fail_setup + mov $192, %eax + xor %ebx, %ebx + mov $65536, %ecx + mov $3, %edx + mov $0x22, %esi + mov $-1, %edi + xor %ebp, %ebp + int $0x80 + cmp $-4095, %eax + jae fail_setup + add $16383, %eax + and $-16384, %eax + mov %eax, page + movl $0x12345678, (%eax) + + movl $0xad7, expected_flags + movl $0x8d5, flag_mask + movl $plain_fault, expected_pc + call arm_fault + call plain + call check_result + run_case cmp8, cmp, 8, 0x8d5 + run_case cmp16, cmp, 16, 0x8d5 + run_case cmp32, cmp, 32, 0x8d5 + run_case test8, test, 8, 0x8c5 + run_case test16, test, 16, 0x8c5 + run_case test32, test, 32, 0x8c5 + run_case same8, test, 8, 0x8c5, 1 + run_case same16, test, 16, 0x8c5, 1 + run_case same32, test, 32, 0x8c5, 1 + run_case bt16, bt, 16, 1 + run_case bt32, bt, 32, 1 + xor %ebx, %ebx + jmp exit + +arm_fault: + movl $0, seen + mov $125, %eax + mov page, %ebx + mov $16384, %ecx + xor %edx, %edx + int $0x80 + test %eax, %eax + js fail_setup + pushl $0xad7 + popfl + mov $0x80008080, %eax + mov $0x7fff7f81, %ebx + mov $0x55667788, %ecx + mov page, %edi + ret + +check_result: + pushfl + pop %edx + xor expected_flags, %edx + and flag_mask, %edx + jnz fail_flags + cmpl $1, seen + jne fail_count + cmp $0x12345678, %ecx + jne fail_state + ret + +.p2align 4 +plain: + nop +plain_fault: + mov (%edi), %ecx + ret + fault_function cmp8, cmp, 8 + fault_function cmp16, cmp, 16 + fault_function cmp32, cmp, 32 + fault_function test8, test, 8 + fault_function test16, test, 16 + fault_function test32, test, 32 + fault_function same8, test, 8, 1 + fault_function same16, test, 16, 1 + fault_function same32, test, 32, 1 + fault_function bt16, bt, 16 + fault_function bt32, bt, 32 + +handler: + cmpl $11, 4(%esp) + jne fail_signal + cmpl $0, seen + jne fail_count + mov 8(%esp), %esi + mov 12(%esi), %eax + cmp page, %eax + jne fail_address + mov 12(%esp), %esi + mov 76(%esi), %eax + cmp expected_pc, %eax + jne fail_pc + mov 84(%esi), %eax + xor expected_flags, %eax + and flag_mask, %eax + jnz fail_flags + cmpl $0x80008080, 64(%esi) + jne fail_state + cmpl $0x7fff7f81, 52(%esi) + jne fail_state + cmpl $0x55667788, 60(%esi) + jne fail_state + mov 36(%esi), %eax + cmp page, %eax + jne fail_state + movl $1, seen + mov $125, %eax + mov page, %ebx + mov $16384, %ecx + mov $3, %edx + int $0x80 + test %eax, %eax + js fail_setup + ret + +restorer: + mov $173, %eax + int $0x80 + ud2 + +fail_setup: mov $1, %ebx; jmp exit +fail_signal: mov $2, %ebx; jmp exit +fail_address: mov $3, %ebx; jmp exit +fail_pc: mov $4, %ebx; jmp exit +fail_flags: mov $5, %ebx; jmp exit +fail_state: mov $6, %ebx; jmp exit +fail_count: mov $7, %ebx +exit: + mov $1, %eax + int $0x80 + +.data +.p2align 14 +action: .long handler, 0x04000004, restorer, 0, 0 +page: .long 0 +expected_pc: .long 0 +expected_flags: .long 0 +flag_mask: .long 0 +seen: .long 0 +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/instptn-signal-recovery.S b/tests/integration/instptn-signal-recovery.S new file mode 100644 index 0000000000..346df989d6 --- /dev/null +++ b/tests/integration/instptn-signal-recovery.S @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +.macro compare kind, bits, same=0 +.if \bits == 8 +.if \same + \kind %al, %al +.else + \kind %bl, %al +.endif +.elseif \bits == 16 +.if \same + \kind %ax, %ax +.else + \kind %bx, %ax +.endif +.elseif \bits == 32 +.if \same + \kind %eax, %eax +.else + \kind %ebx, %eax +.endif +.else +.if \same + \kind %rax, %rax +.else + \kind %rbx, %rax +.endif +.endif +.endm + +.macro run_case name, kind, bits, mask, same=0 + movabs $0x8000000080008080, %rax + movabs $0x7fffffff7fff7f81, %rbx +.ifc \kind,bt + compare bt, \bits +.else + compare \kind, \bits, \same +.endif + pushfq + popq expected_flags(%rip) + movq $\mask, flag_mask(%rip) + lea \name\()_fault(%rip), %rax + mov %rax, expected_pc(%rip) + call arm_fault + call \name + call check_result +.endm + +.macro fault_function name, kind, bits, same=0 +.p2align 4 +\name: +.ifc \kind,bt + compare bt, \bits +.else + compare \kind, \bits, \same +.endif +\name\()_fault: + mov (%rdi), %rcx +.ifc \kind,bt + jc 1f +.else + jz 1f +.endif + ret +1: + ret +.endm + +.text +.globl _start +_start: + mov $13, %eax + mov $11, %edi + lea action(%rip), %rsi + xor %edx, %edx + mov $8, %r10d + syscall + test %rax, %rax + js fail_setup + /* Shared mappings select the real parallel recovery path even without + * another thread. It must not depend on a fresh push/pop retranslation. */ + mov $9, %eax + xor %edi, %edi + mov $16384, %esi + mov $3, %edx + mov $0x21, %r10d + mov $-1, %r8 + xor %r9d, %r9d + syscall + test %rax, %rax + js fail_setup + mov $9, %eax + xor %edi, %edi + mov $65536, %esi + mov $3, %edx + mov $0x22, %r10d + mov $-1, %r8 + xor %r9d, %r9d + syscall + test %rax, %rax + js fail_setup + add $16383, %rax + and $-16384, %rax + mov %rax, page(%rip) + movq $0x12345678, (%rax) + movdqu poison(%rip), %xmm0 + + /* No delayed comparison: hot AOT must not consult a temporary IR1 list. */ + movq $0xad7, expected_flags(%rip) + movq $0x8d5, flag_mask(%rip) + lea plain_fault(%rip), %rax + mov %rax, expected_pc(%rip) + call arm_fault + call plain + call check_result + + run_case cmp8, cmp, 8, 0x8d5 + run_case cmp16, cmp, 16, 0x8d5 + run_case cmp32, cmp, 32, 0x8d5 + run_case cmp64, cmp, 64, 0x8d5 + run_case test8, test, 8, 0x8c5 + run_case test16, test, 16, 0x8c5 + run_case test32, test, 32, 0x8c5 + run_case test64, test, 64, 0x8c5 + run_case same8, test, 8, 0x8c5, 1 + run_case same16, test, 16, 0x8c5, 1 + run_case same32, test, 32, 0x8c5, 1 + run_case same64, test, 64, 0x8c5, 1 + run_case bt16, bt, 16, 1 + run_case bt32, bt, 32, 1 + run_case bt64, bt, 64, 1 + + xor %edi, %edi + jmp exit + +/* Protect a whole host page; no shared-page interpretation is involved. */ +arm_fault: + movl $0, seen(%rip) + mov $10, %eax + mov page(%rip), %rdi + mov $16384, %esi + xor %edx, %edx + syscall + test %rax, %rax + js fail_setup + pushq $0xad7 + popfq + movabs $0x8000000080008080, %rax + movabs $0x7fffffff7fff7f81, %rbx + movabs $0x1122334455667788, %rcx + ret + +check_result: + pushfq + pop %r8 + xor expected_flags(%rip), %r8 + and flag_mask(%rip), %r8 + jnz fail_flags + cmpl $1, seen(%rip) + jne fail_count + cmp $0x12345678, %rcx + jne fail_state + movdqu %xmm0, actual(%rip) + mov actual(%rip), %r8 + cmp poison(%rip), %r8 + jne fail_state + mov actual+8(%rip), %r8 + cmp poison+8(%rip), %r8 + jne fail_state + ret + +.p2align 4 +plain: + nop +plain_fault: + mov (%rdi), %rcx + ret + + fault_function cmp8, cmp, 8 + fault_function cmp16, cmp, 16 + fault_function cmp32, cmp, 32 + fault_function cmp64, cmp, 64 + fault_function test8, test, 8 + fault_function test16, test, 16 + fault_function test32, test, 32 + fault_function test64, test, 64 + fault_function same8, test, 8, 1 + fault_function same16, test, 16, 1 + fault_function same32, test, 32, 1 + fault_function same64, test, 64, 1 + fault_function bt16, bt, 16 + fault_function bt32, bt, 32 + fault_function bt64, bt, 64 + +handler: + cmp $11, %edi + jne fail_signal + cmpl $0, seen(%rip) + jne fail_count + mov 16(%rsi), %r8 + cmp page(%rip), %r8 + jne fail_address + mov 168(%rdx), %r8 + cmp expected_pc(%rip), %r8 + jne fail_pc + mov 176(%rdx), %r8 + xor expected_flags(%rip), %r8 + and flag_mask(%rip), %r8 + jnz fail_flags + movabs $0x8000000080008080, %r8 + cmp 144(%rdx), %r8 + jne fail_state + movabs $0x7fffffff7fff7f81, %r8 + cmp 128(%rdx), %r8 + jne fail_state + movabs $0x1122334455667788, %r8 + cmp 152(%rdx), %r8 + jne fail_state + mov 104(%rdx), %r8 + cmp page(%rip), %r8 + jne fail_state + mov 224(%rdx), %r9 + mov 160(%r9), %r8 + cmp poison(%rip), %r8 + jne fail_state + mov 168(%r9), %r8 + cmp poison+8(%rip), %r8 + jne fail_state + movl $1, seen(%rip) + mov $10, %eax + mov page(%rip), %rdi + mov $16384, %esi + mov $3, %edx + syscall + test %rax, %rax + js fail_setup + ret + +restorer: + mov $15, %eax + syscall + ud2 + +fail_setup: mov $1, %edi; jmp exit +fail_signal: mov $2, %edi; jmp exit +fail_address: mov $3, %edi; jmp exit +fail_pc: mov $4, %edi; jmp exit +fail_flags: mov $5, %edi; jmp exit +fail_state: mov $6, %edi; jmp exit +fail_count: mov $7, %edi +exit: + mov $60, %eax + syscall + +.data +/* Keep mutable test state out of the code's 16 KiB host page. */ +.p2align 14 +action: .quad handler, 0x04000004, restorer, 0 +poison: .quad 0x0123456789abcdef, 0xfedcba9876543210 +page: .quad 0 +expected_pc: .quad 0 +expected_flags: .quad 0 +flag_mask: .quad 0 +seen: .long 0 +.bss +.p2align 4 +actual: .skip 16 +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index 02e1e2656c..7e56ef9460 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -1,5 +1,32 @@ +if 'i386-linux-user' in target_dirs + latx_integration_tests += [{ + 'name': 'test-ucomisd-seta-i386', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-i386'], '--guest-bits', '32', + files('../../ucomisd-seta-i386.asm')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 180, + }] +endif + if 'x86_64-linux-user' in target_dirs + foreach cvt_option : ['0', '1'] + latx_integration_tests += [{ + 'name': 'test-avx-conversion-alias-' + cvt_option, + 'runner': find_program('../../test-avx-conversion-alias.py'), + 'args': [emulators['latx-x86_64'], files('../../vcvtpd-register-alias.asm')], + 'env': {'LATX_SOFTFPU': '0', 'LATX_CVT_OPT': cvt_option}, + 'timeout': 120, + }] + endforeach if 'CONFIG_LATX_AVX_OPT' in config_host + latx_integration_tests += [{ + 'name': 'test-vcmpsd-writeback', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], files('../../vcmpsd-writeback.asm')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 120, + }] latx_integration_tests += [{ 'name': 'test-f16c-memory-boundary', 'runner': find_program('../../test-avx-regressions.py'), @@ -19,6 +46,14 @@ if 'x86_64-linux-user' in target_dirs 'timeout': 120, }] endif + latx_integration_tests += [{ + 'name': 'test-instptn-signal-recovery', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], + files('../../instptn-signal-recovery.S')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 120, + }] latx_integration_tests += [{ 'name': 'test-ymmh-conditional-save', 'runner': find_program('../../test-avx-regressions.py'), @@ -44,6 +79,48 @@ if 'x86_64-linux-user' in target_dirs 'args': [emulators['latx-x86_64'], files('../../vcomis-jcc.S')], 'timeout': 120, }] + latx_integration_tests += [{ + 'name': 'test-ucomisd-seta', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], files('../../ucomisd-seta.asm')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 180, + }] + latx_integration_tests += [{ + 'name': 'test-ucomisd-seta-fault', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], files('../../ucomisd-seta-fault.asm')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 180, + }] + latx_integration_tests += [{ + 'name': 'test-ucomisd-seta-fpe', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], files('../../ucomisd-seta-fpe.asm')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 180, + }] + latx_integration_tests += [{ + 'name': 'test-vcomis-flag-reduction', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], files('../../vcomis-flag-reduction.asm')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 180, + }] + latx_integration_tests += [{ + 'name': 'test-avx-test-flags', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], files('../../avx-test-flags.asm')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 180, + }] + latx_integration_tests += [{ + 'name': 'test-avx-test-jcc', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], files('../../avx-test-jcc.asm')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 180, + }] latx_integration_tests += [{ 'name': 'test-xcomis-patterns', 'runner': find_program('../../test-avx-regressions.py'), @@ -74,10 +151,28 @@ if 'x86_64-linux-user' in target_dirs 'args': [ emulators['latx-x86_64'], files('../../avx-single-instruction-boundaries.asm'), + files('../../avx-scalar-double-alias.asm'), + files('../../vaddsubpd-lanes.asm'), + files('../../vblendpd-xmm-alias.asm'), + files('../../vbroadcastsd-memory.asm'), + files('../../vinsertf128-alias.asm'), + files('../../vinsertf128-high-pending.asm'), + files('../../vmovsd-alias.asm'), + files('../../vperm2x128-immediate.asm'), + files('../../vpermilpd-ymm-immediate.asm'), + files('../../vpermilpd-immediate.asm'), + files('../../vmovd-xmm-zero.asm'), + files('../../vmovss-memory-zero.asm'), files('../../avx128-shuffle.S'), files('../../avx-ymm-writeback.S'), files('../../vpshufd-distinct-regs.S'), files('../../vroundps-imm0b.S'), + files('../../vroundsd-fixed.asm'), + files('../../vroundss-fixed.asm'), + files('../../vcvtss2sd-alias.asm'), + files('../../vcvtdq2pd-ymm-alias.asm'), + files('../../vcvtsi2sd-alias.asm'), + files('../../vshufpd-interleave.asm'), files('../../avx-regression-extract-alias.S'), files('../../avx-regression-narrow-state.S'), files('../../avx-regression-fma-alias.S'), @@ -97,6 +192,14 @@ if 'x86_64-linux-user' in target_dirs suite: 'latx-integration', ) foreach cache_option : ['0000', '1100', '1101', '1110', '1111'] + latx_integration_tests += [{ + 'name': 'test-base-displacement-' + cache_option, + 'runner': find_program('../../test-avx-regressions.py'), + 'args': [emulators['latx-x86_64'], + files('../../base-displacement.asm')], + 'env': {'LATX_IMM_REG': cache_option, 'LATX_SOFTFPU': '0'}, + 'timeout': 120, + }] test( 'test-rip-relative-cache-' + cache_option, find_program('../../test-avx-regressions.py'), @@ -126,6 +229,14 @@ if 'i386-linux-user' in target_dirs 'timeout': 120, }] endif + latx_integration_tests += [{ + 'name': 'test-instptn-signal-recovery-i386', + 'runner': find_program('../../test-avx-regressions.py'), + 'args': ['--guest-bits', '32', emulators['latx-i386'], + files('../../instptn-signal-recovery-i386.S')], + 'env': {'LATX_SOFTFPU': '0'}, + 'timeout': 120, + }] latx_integration_tests += [{ 'name': 'test-ymmh-conditional-save-i386', 'runner': find_program('../../test-avx-regressions.py'), diff --git a/tests/integration/test-avx-conversion-alias.py b/tests/integration/test-avx-conversion-alias.py new file mode 100755 index 0000000000..0dee8ccfe1 --- /dev/null +++ b/tests/integration/test-avx-conversion-alias.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 LAT Project Authors +# SPDX-License-Identifier: GPL-2.0-only +"""Reject unsupported-register diagnostics as well as wrong AVX results.""" +from pathlib import Path +import subprocess +import sys + +runner = Path(__file__).with_name("test-avx-regressions.py") +result = subprocess.run([sys.executable, str(runner), *sys.argv[1:]], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True) +print(result.stdout, end="") +if "reg no match" in result.stdout: + print("FAIL: conversion used a register check that does not support its operands") + sys.exit(1) +sys.exit(result.returncode) diff --git a/tests/integration/ucomisd-seta-fault.asm b/tests/integration/ucomisd-seta-fault.asm new file mode 100644 index 0000000000..310331c163 --- /dev/null +++ b/tests/integration/ucomisd-seta-fault.asm @@ -0,0 +1,208 @@ +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +%macro ARM 2 + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + xor edx, edx + syscall + test eax, eax + jne fail_setup + lea rax, [%1] + mov [expected_pc], rax + mov qword [expected_flags], %2 + mov dword [seen], 0 + vmovdqu ymm0, [two] + vmovdqu ymm1, [one] + mov r8, 0x1122334455667788 + push qword 0xad7 + popfq +%endmacro + +%macro DONE 0 + cmp dword [seen], 1 + jne fail_count + cmp r8b, 1 + jne fail_result + vmovdqu [actual], ymm0 + vmovdqu [actual + 32], ymm1 +%assign i 0 +%rep 4 + mov rax, [actual + i * 8] + cmp rax, [two + i * 8] + jne fail_state + mov rax, [actual + 32 + i * 8] + cmp rax, [one + i * 8] + jne fail_state +%assign i i + 1 +%endrep +%endmacro + +%macro TESTS 1 + ; A fault loading the comparison source must retain pre-comparison flags. + mov dword [stage], 1 + ARM %%load, 0x8d5 +%%load: + %1 xmm0, [r13] + seta r8b + xor ecx, ecx + DONE + + ; A later fault must expose comparison flags, even if XOR kills them. + mov dword [stage], 2 + ARM %%after, 0 + %1 xmm0, xmm1 + seta r8b +%%after: + mov rax, [r13] + xor ecx, ecx + DONE + + ; A memory SETA needs the new flags and the SETA PC on a store fault. + mov dword [stage], 3 + ARM %%store, 0 + %1 xmm0, xmm1 +%%store: + seta byte [r13] + mov r8b, [r13] + xor ecx, ecx + DONE + + ; The source operand is only eight bytes, ending at a protected page. + ARM %%unused, 0 + %1 xmm0, [r13 - 8] + seta r8b + xor ecx, ecx + cmp dword [seen], 0 + jne fail_count + cmp r8b, 1 + jne fail_result + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + mov edx, 3 + syscall +%%unused: +%endmacro + +section .text +_start: + ldmxcsr [clean] + mov eax, 13 + mov edi, 11 + lea rsi, [action] + xor edx, edx + mov r10d, 8 + syscall + test eax, eax + jne fail_setup + mov eax, 9 + xor edi, edi + mov esi, 131072 + mov edx, 3 + mov r10d, 0x22 + mov r8, -1 + xor r9d, r9d + syscall + cmp rax, -4095 + jae fail_setup + mov r12, rax + lea r13, [rax + 65536] + mov rax, [one] + mov [r13], rax + mov [r13 - 8], rax + TESTS ucomisd + TESTS vucomisd + mov eax, 11 + mov rdi, r12 + mov esi, 131072 + syscall + xor edi, edi + jmp exit + +handler: + cmp edi, 11 + jne fail_signal + cmp dword [seen], 0 + jne fail_count + cmp [rsi + 16], r13 + jne fail_address + mov rax, [rdx + 168] + cmp rax, [expected_pc] + jne fail_pc + mov rax, [rdx + 176] + and eax, 0x8d5 + cmp rax, [expected_flags] + jne fail_flags + mov r9, [rdx + 224] +%assign i 0 +%rep 2 + mov rax, [r9 + 160 + i * 8] + cmp rax, [two + i * 8] + jne fail_state + mov rax, [r9 + 176 + i * 8] + cmp rax, [one + i * 8] + jne fail_state + mov rax, [r9 + 576 + i * 8] + cmp rax, [two + 16 + i * 8] + jne fail_state + mov rax, [r9 + 592 + i * 8] + cmp rax, [one + 16 + i * 8] + jne fail_state +%assign i i + 1 +%endrep + inc dword [seen] + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + mov edx, 3 + syscall + test eax, eax + jne fail_setup + ret +restorer: + mov eax, 15 + syscall + ud2 +fail_setup: mov edi, 1 + jmp exit +fail_signal: mov edi, 2 + jmp exit +fail_count: mov edi, 3 + jmp exit +fail_address: mov edi, 4 + jmp exit +fail_pc: + mov edi, [stage] + imul edi, edi, 10 + add edi, 5 + jmp exit +fail_flags: + mov edi, [stage] + imul edi, edi, 10 + add edi, 6 + jmp exit +fail_state: mov edi, 7 + jmp exit +fail_result: mov edi, 8 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +two: dq 0x4000000000000000, 0x0123456789abcdef + dq 0x1122334455667788, 0xfedcba9876543210 +one: dq 0x3ff0000000000000, 0x9999888877776666 + dq 0xaabbccddeeff0011, 0x123456789abcdef0 +action: dq handler, 0x04000004, restorer, 0 +clean: dd 0x1f80 +section .bss align=32 +actual: resb 64 +expected_pc: resq 1 +expected_flags: resq 1 +seen: resd 1 +stage: resd 1 +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/ucomisd-seta-fpe.asm b/tests/integration/ucomisd-seta-fpe.asm new file mode 100644 index 0000000000..47d6a6ae37 --- /dev/null +++ b/tests/integration/ucomisd-seta-fpe.asm @@ -0,0 +1,190 @@ +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +%macro CASE 1 + mov dword [seen], 0 + mov dword [expected_set], 0x40 + lea rax, [%%trap] + mov [expected_pc], rax + lea rax, [%%resume] + mov [resume_pc], rax + movsd xmm0, [one] + movsd xmm1, [snan] + mov eax, 1 + test eax, eax + pushfq + pop rdi + %1 xmm0, xmm0 + seta al + setc r8b + ; This flag write can be removed because the next comparison defines ZF. + xor ecx, ecx +%%trap: + %1 xmm1, xmm0 +%%resume: + setz r9b + xor ecx, ecx + cmp r9b, 1 + jne fail_flags + cmp r8b, 0 + jne fail_result + cmp dword [seen], 1 + jne fail_count +%endmacro + +%macro ONLY_OVERFLOW_LIVE 1 + mov dword [seen], 0 + mov dword [expected_set], 0x8d5 + lea rax, [%%trap] + mov [expected_pc], rax + lea rax, [%%resume] + mov [resume_pc], rax + movsd xmm0, [one] + movsd xmm1, [snan] + push qword 0x8d7 + popfq + jmp %%trap +%%trap: + %1 xmm1, xmm0 +%%resume: + seta al + seto r8b + cmp ecx, ecx + pushfq + pop rcx + test al, al + jne fail_result + cmp r8b, 1 + jne fail_flags + cmp dword [seen], 1 + jne fail_count +%endmacro + +%macro SNAN_MASK 3 + mov dword [seen], 0 + mov dword [expected_set], 0x8d5 + lea rax, [%%trap] + mov [expected_pc], rax + lea rax, [%%resume] + mov [resume_pc], rax + ldmxcsr [mxcsr] + movsd xmm0, [snan] + movsd xmm1, [one] + push qword 0x8d7 + popfq + jmp %%trap +%%trap: +%if %2 = 0 + %1 xmm0, xmm1 +%elif %2 = 1 + %1 xmm0, xmm0 +%else + %1 xmm0, [one] +%endif +%%resume: + seta al +%if %3 & 1 + setc r8b +%endif +%if %3 & 2 + setp r9b +%endif +%if %3 & 4 + lahf +%endif +%if %3 & 8 + setz r10b +%endif +%if %3 & 16 + sets r11b +%endif +%if %3 & 32 + seto r12b +%endif + cmp ecx, ecx + pushfq + pop rcx + test al, al + jne fail_result + cmp dword [seen], 1 + jne fail_count +%endmacro + +section .text +_start: + mov eax, 13 + mov edi, 8 + lea rsi, [action] + xor edx, edx + mov r10d, 8 + syscall + test eax, eax + jne fail_setup + ; Both x87 and SSE invalid-operation traps are enabled, not soft float. + fldcw [control] + ldmxcsr [mxcsr] + CASE ucomisd + CASE vucomisd + ONLY_OVERFLOW_LIVE ucomisd + ONLY_OVERFLOW_LIVE vucomisd +%assign mask 0 +%rep 64 +%assign form 0 +%rep 3 + SNAN_MASK ucomisd, form, mask + SNAN_MASK vucomisd, form, mask +%assign form form + 1 +%endrep +%assign mask mask + 1 +%endrep + xor edi, edi + jmp exit +handler: + cmp edi, 8 + jne fail_signal + mov rax, [rdx + 168] + cmp rax, [expected_pc] + jne fail_pc + mov eax, [rdx + 176] + and eax, [expected_set] + cmp eax, [expected_set] + jne fail_flags + cmp dword [seen], 0 + jne fail_count + inc dword [seen] + ; Skip the deliberate fault, as the Lative random-test handler does. + mov rax, [resume_pc] + mov [rdx + 168], rax + ret +restorer: + mov eax, 15 + syscall + ud2 +fail_setup: mov edi, 1 + jmp exit +fail_signal: mov edi, 2 + jmp exit +fail_pc: mov edi, 3 + jmp exit +fail_flags: mov edi, 4 + jmp exit +fail_result: mov edi, 5 + jmp exit +fail_count: mov edi, 6 +exit: + mov eax, 60 + syscall +section .rodata +one: dq 0x3ff0000000000000 +snan: dq 0x7ff0000000000001 +control: dw 0x037e +mxcsr: dd 0x1f00 +action: dq handler, 0x04000004, restorer, 0 +section .bss +expected_pc: resq 1 +resume_pc: resq 1 +seen: resd 1 +expected_set: resd 1 +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/ucomisd-seta-i386.asm b/tests/integration/ucomisd-seta-i386.asm new file mode 100644 index 0000000000..136c82908c --- /dev/null +++ b/tests/integration/ucomisd-seta-i386.asm @@ -0,0 +1,115 @@ +; SPDX-License-Identifier: GPL-2.0-only +bits 32 +global _start + +%macro CASE 6 + mov eax, 0x12345678 + mov ebx, eax + mov ecx, eax + mov edx, eax +%if %2 = 0 + %1 xmm0, xmm1 +%elif %2 = 1 + %1 xmm0, xmm0 +%else + %1 xmm0, [esi + 8] +%endif + seta %3 + mov ebp, %4 +%if %6 + pushfd + pop edx +%else + ; Keep this full flag write so the compare flags are not needed. + cmp ebp, ebp + pushfd + pop edx +%endif + mov edi, [esi + 16] +%if %2 = 1 + mov edi, [esi + 20] +%endif +%if %6 + and edx, 0x8d5 + cmp edx, edi + jne fail_flags +%endif + mov eax, edi + and eax, 0x41 + sete al + movzx eax, al +%if %5 = 8 + shl eax, 8 + or eax, 0x12340078 +%else + or eax, 0x12345600 +%endif + cmp ebp, eax + jne fail_result +%endmacro + +%macro DESTINATIONS 3 + CASE %1, %2, al, eax, 0, %3 + CASE %1, %2, ah, eax, 8, %3 + CASE %1, %2, bl, ebx, 0, %3 + CASE %1, %2, bh, ebx, 8, %3 + CASE %1, %2, cl, ecx, 0, %3 + CASE %1, %2, ch, ecx, 8, %3 + CASE %1, %2, dl, edx, 0, %3 + CASE %1, %2, dh, edx, 8, %3 +%endmacro + +section .text +_start: + ldmxcsr [clean] + mov esi, pairs +next_pair: + movsd xmm0, [esi] + movsd xmm1, [esi + 8] + call suite + add esi, 24 + cmp esi, pairs_end + jne next_pair + xor ebx, ebx + jmp exit +suite: +%assign live 0 +%rep 2 +%assign form 0 +%rep 3 + DESTINATIONS ucomisd, form, live + DESTINATIONS vucomisd, form, live +%assign form form + 1 +%endrep +%assign live live + 1 +%endrep + ret +fail_result: mov ebx, 1 + jmp exit +fail_flags: mov ebx, 2 +exit: + mov eax, 1 + int 0x80 + +%macro PAIR 4 + dq %1, %2 + dd %3, %4 +%endmacro +section .rodata align=8 +clean: dd 0x1f80 +align 8 +pairs: + PAIR 0x4000000000000000, 0x3ff0000000000000, 0, 0x40 + PAIR 0x3ff0000000000000, 0x4000000000000000, 1, 0x40 + PAIR 0x3ff0000000000000, 0x3ff0000000000000, 0x40, 0x40 + PAIR 0, 0x8000000000000000, 0x40, 0x40 + PAIR 0x7ff0000000000000, 0x4000000000000000, 0, 0x40 + PAIR 0xfff0000000000000, 0xbff0000000000000, 1, 0x40 + PAIR 0x7ff8000000000001, 0x3ff0000000000000, 0x45, 0x45 + PAIR 0x3ff0000000000000, 0xfff800000000cafe, 0x45, 0x40 + PAIR 0x7ff0000000000001, 0x3ff0000000000000, 0x45, 0x45 + PAIR 0x3ff0000000000000, 0xfff0000000000001, 0x45, 0x40 + PAIR 1, 0, 0, 0x40 + PAIR 0x8000000000000001, 0, 1, 0x40 +pairs_end: +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/ucomisd-seta.asm b/tests/integration/ucomisd-seta.asm new file mode 100644 index 0000000000..faaba6097a --- /dev/null +++ b/tests/integration/ucomisd-seta.asm @@ -0,0 +1,185 @@ +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +%macro CHECK_FLAG 4 +%if %4 & (1 << %1) + mov eax, ebp + shr eax, %3 + and eax, 1 + cmp %2, al + jne fail_flags +%endif +%endmacro + +%macro CASE 4 + vmovdqu ymm0, [rsi] + vmovdqu ymm1, [rsi + 32] + mov ebp, [rsi + 64] +%if %2 = 1 + mov ebp, [rsi + 68] +%endif + mov rax, 0x123456789abcde00 + mov r8, rax +%if %2 = 0 + %1 xmm0, xmm1 +%elif %2 = 1 + %1 xmm0, xmm0 +%else + %1 xmm0, [rsi + 32] +%endif +%if %3 = 0 + seta al + mov rdx, rax +%elif %3 = 1 + seta ah + mov rdx, rax +%else + seta r8b + mov rdx, r8 +%endif +%if %4 & 1 + setc r9b +%endif +%if %4 & 2 + setp r10b +%endif +%if %4 & 4 + lahf + movzx ebx, ah + mov r14d, ebx +%endif +%if %4 & 8 + setz r11b +%endif +%if %4 & 16 + sets r12b +%endif +%if %4 & 32 + seto r13b +%endif + cmp ecx, ecx + pushfq + pop rcx + mov ecx, 0 + mov eax, ebp + and eax, 0x41 + sete cl +%if %3 = 1 + shl ecx, 8 + mov rbx, 0x123456789abc0000 +%else + mov rbx, 0x123456789abcde00 +%endif + or rbx, rcx + cmp rdx, rbx + jne fail_result + CHECK_FLAG 0, r9b, 0, %4 + CHECK_FLAG 1, r10b, 2, %4 +%if %4 & 4 + mov eax, ebp + or eax, 2 + cmp r14b, al + jne fail_flags +%endif + CHECK_FLAG 3, r11b, 6, %4 + CHECK_FLAG 4, r12b, 7, %4 + CHECK_FLAG 5, r13b, 11, %4 + vmovdqu [actual], ymm0 + vmovdqu [actual + 32], ymm1 +%assign q 0 +%rep 8 + mov rax, [actual + q * 8] + cmp rax, [rsi + q * 8] + jne fail_vector +%assign q q + 1 +%endrep +%endmacro + +%macro STACK_BYTE 5 + mov rdi, %3 + mov rdx, rdi + and rdx, -256 + %1 xmm0, %4 + seta %2 + mov rcx, %3 + mov %3, rdi + cmp eax, eax + pushfq + pop rax + or rdx, %5 + cmp rcx, rdx + jne fail_result +%endmacro + +section .text +_start: + ldmxcsr [clean] + lea rsi, [pairs] + mov r15d, (pairs_end - pairs) / 96 +next_pair: + call suite + add rsi, 96 + dec r15d + jnz next_pair + ; The random-test harness reserves RSP and RBP. + movsd xmm0, [pairs] + movsd xmm1, [pairs + 32] + STACK_BYTE ucomisd, spl, rsp, xmm1, 1 + STACK_BYTE ucomisd, bpl, rbp, xmm0, 0 + STACK_BYTE vucomisd, spl, rsp, xmm0, 0 + STACK_BYTE vucomisd, bpl, rbp, xmm1, 1 + xor edi, edi + jmp exit +suite: +%assign mask 0 +%rep 64 +%assign form 0 +%rep 3 +%assign dest 0 +%rep 3 + CASE ucomisd, form, dest, mask + CASE vucomisd, form, dest, mask +%assign dest dest + 1 +%endrep +%assign form form + 1 +%endrep +%assign mask mask + 1 +%endrep + ret +fail_result: mov edi, 1 + jmp exit +fail_flags: mov edi, 2 + jmp exit +fail_vector: mov edi, 3 +exit: + mov eax, 60 + syscall + +%macro PAIR 4 + dq %1, 0x0123456789abcdef, 0xffeeddccbbaa9988, 0x1122334455667788 + dq %2, 0xfedcba9876543210, 0xaabbccddeeff0011, 0x8877665544332211 + dd %3, %4 + times 24 db 0 +%endmacro +section .rodata align=32 +clean: dd 0x1f80 +align 32 +pairs: + PAIR 0x4000000000000000, 0x3ff0000000000000, 0, 0x40 + PAIR 0x3ff0000000000000, 0x4000000000000000, 1, 0x40 + PAIR 0x3ff0000000000000, 0x3ff0000000000000, 0x40, 0x40 + PAIR 0, 0x8000000000000000, 0x40, 0x40 + PAIR 0x7ff0000000000000, 0x4000000000000000, 0, 0x40 + PAIR 0xfff0000000000000, 0xbff0000000000000, 1, 0x40 + PAIR 0x7ff8000000000001, 0x3ff0000000000000, 0x45, 0x45 + PAIR 0x3ff0000000000000, 0xfff800000000cafe, 0x45, 0x40 + PAIR 0x7ff0000000000001, 0x3ff0000000000000, 0x45, 0x45 + PAIR 0x3ff0000000000000, 0xfff0000000000001, 0x45, 0x40 + PAIR 1, 0, 0, 0x40 + PAIR 0x8000000000000001, 0, 1, 0x40 +pairs_end: +section .bss align=32 +actual: resb 64 +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vaddsubpd-lanes.asm b/tests/integration/vaddsubpd-lanes.asm new file mode 100644 index 0000000000..c4d8fcbb5b --- /dev/null +++ b/tests/integration/vaddsubpd-lanes.asm @@ -0,0 +1,86 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +section .text + +%macro CHECK 2 + mov edi, %2 + vmovdqu [actual], ymm0 + mov rax, [actual] + cmp rax, [%1] + jne fail + mov rax, [actual + 8] + cmp rax, [%1 + 8] + jne fail + mov rax, [actual + 16] + or rax, [actual + 24] + jne fail +%endmacro + +%macro ALIASES 0 + vmovdqu ymm0, [junk] + vmovdqu ymm1, [input_a] + vmovdqu ymm2, [input_b] + vaddsubpd xmm0, xmm1, xmm2 + CHECK expected_ab, 1 + vmovdqu ymm0, [input_a] + vaddsubpd xmm0, xmm0, xmm2 + CHECK expected_ab, 2 + vmovdqu ymm0, [input_b] + vaddsubpd xmm0, xmm1, xmm0 + CHECK expected_ab, 3 + vaddsubpd xmm0, xmm1, xmm1 + CHECK rbx, 4 + vmovdqu ymm0, [input_a] + vaddsubpd xmm0, xmm0, xmm0 + CHECK rbx, 5 + vaddsubpd xmm0, xmm1, [input_b] + CHECK expected_ab, 6 + vmovdqu ymm0, [input_a] + vaddsubpd xmm0, xmm0, [input_b] + CHECK expected_ab, 7 +%endmacro + +_start: + mov ecx, 4 +.rounding: + lea rbx, [expected_aa] + cmp ecx, 2 + jne .expected_ready + lea rbx, [expected_aa_down] +.expected_ready: + mov eax, ecx + dec eax + shl eax, 13 + or eax, 0x1f80 + mov [mxcsr], eax + ldmxcsr [mxcsr] + ALIASES + ; Unused YMM lanes contain signaling NaNs and must not raise invalid. + stmxcsr [mxcsr] + mov edi, 8 + test dword [mxcsr], 1 + jnz fail + dec ecx + jnz .rounding + xor edi, edi +fail: + mov eax, 60 + syscall + +section .rodata align=32 +input_a: dq 6.0, -4.0, 0x7ff0000000000001, 0x7ff0000000000001 +input_b: dq 2.0, 1.0, 0x7ff0000000000001, 0x7ff0000000000001 +junk: times 4 dq 0xdeadbeefdeadbeef +expected_ab: dq 4.0, -3.0 +expected_aa: dq 0.0, -8.0 +expected_aa_down: dq 0x8000000000000000, -8.0 + +section .bss align=32 +actual: resb 32 +mxcsr: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vblendpd-xmm-alias.asm b/tests/integration/vblendpd-xmm-alias.asm new file mode 100644 index 0000000000..b747b297cb --- /dev/null +++ b/tests/integration/vblendpd-xmm-alias.asm @@ -0,0 +1,109 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro CHECK_RESULT 1 + vmovdqu [actual], ymm0 + lea rdx, [%1] + mov ecx, 4 +%%check: + mov rax, [actual + rcx * 8 - 8] + cmp rax, [rdx + rcx * 8 - 8] + jne fail + dec ecx + jnz %%check +%endmacro + +%macro PREPARE 1 + vmovdqu ymm1, [src1] + vmovdqu ymm2, [src2] +%if %1 == 0 + vmovdqu ymm0, [src2] +%endif +%endmacro + +_start: + PREPARE 0 + vblendpd xmm2, xmm1, xmm2, 0 + vmovdqu ymm0, ymm2 + CHECK_RESULT expected0 + + PREPARE 0 + vblendpd xmm2, xmm1, xmm2, 1 + vmovdqu ymm0, ymm2 + CHECK_RESULT expected1 + + PREPARE 0 + vblendpd xmm2, xmm1, xmm2, 2 + vmovdqu ymm0, ymm2 + CHECK_RESULT expected2 + + PREPARE 0 + vblendpd xmm2, xmm1, xmm2, 3 + vmovdqu ymm0, ymm2 + CHECK_RESULT expected3 + + PREPARE 0 + vblendpd xmm1, xmm1, xmm2, 0 + vmovdqu ymm0, ymm1 + CHECK_RESULT expected0 + + PREPARE 0 + vblendpd xmm1, xmm1, xmm2, 1 + vmovdqu ymm0, ymm1 + CHECK_RESULT expected1 + + PREPARE 0 + vblendpd xmm1, xmm1, xmm2, 2 + vmovdqu ymm0, ymm1 + CHECK_RESULT expected2 + + PREPARE 0 + vblendpd xmm1, xmm1, xmm2, 3 + vmovdqu ymm0, ymm1 + CHECK_RESULT expected3 + + PREPARE 0 + vblendpd xmm1, xmm1, xmm1, 1 + vmovdqu ymm0, ymm1 + CHECK_RESULT expected0 + + PREPARE 1 + vblendpd xmm0, xmm1, xmm2, 1 + CHECK_RESULT expected1 + + PREPARE 1 + vblendpd xmm0, xmm1, [src2], 2 + CHECK_RESULT expected2 + + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +src1: + dq 0x3ff0000000000000, 0x4000000000000000 + dq 0xaaaaaaaaaaaaaaaa, 0xbbbbbbbbbbbbbbbb +src2: + dq 0x4008000000000000, 0x4010000000000000 + dq 0xcccccccccccccccc, 0xdddddddddddddddd +expected0: + dq 0x3ff0000000000000, 0x4000000000000000, 0, 0 +expected1: + dq 0x4008000000000000, 0x4000000000000000, 0, 0 +expected2: + dq 0x3ff0000000000000, 0x4010000000000000, 0, 0 +expected3: + dq 0x4008000000000000, 0x4010000000000000, 0, 0 + +section .bss align=32 +actual: resb 32 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vbroadcastsd-memory.asm b/tests/integration/vbroadcastsd-memory.asm new file mode 100644 index 0000000000..ef7b94cbf8 --- /dev/null +++ b/tests/integration/vbroadcastsd-memory.asm @@ -0,0 +1,163 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +%macro CHECK 0 + vmovdqu [actual], ymm0 + mov rax, [rdi] + lea rsi, [actual] + mov ecx, 4 +%%loop: + cmp [rsi], rax + jne fail + add rsi, 8 + loop %%loop +%endmacro + +%macro LOAD_DISP 1 + lea rbx, [data + 64 - %1] + lea rdi, [rbx + %1] + vbroadcastsd ymm0, [rbx + %1] + CHECK +%endmacro + +section .text +_start: + stmxcsr [original_mxcsr] + vmovdqu ymm1, [sentinel] + LOAD_DISP -4096 + LOAD_DISP -2048 + LOAD_DISP -2040 + LOAD_DISP -9 + LOAD_DISP -8 + LOAD_DISP -1 + LOAD_DISP 0 + LOAD_DISP 1 + LOAD_DISP 8 + LOAD_DISP 2040 + LOAD_DISP 2047 + LOAD_DISP 4096 + +%assign offset 0 +%rep 8 + lea rbx, [data + offset] + lea rdi, [rbx + 8] + vxorps xmm0, xmm0, xmm0 + vbroadcastsd ymm0, [rbx + 8] + CHECK +%assign offset offset + 1 +%endrep + + ; A register source must still work when its upper half is pending zero. + vmovdqu ymm0, [sentinel] + vmovdqa xmm0, [data] + vbroadcastsd ymm0, xmm0 + lea rdi, [data] + CHECK + vbroadcastsd ymm0, xmm1 + lea rdi, [sentinel] + CHECK + vmovdqu [actual], ymm1 + lea rsi, [actual] + lea rdi, [sentinel] + mov ecx, 4 +.preserved: + mov rax, [rsi] + cmp rax, [rdi] + jne fail + add rsi, 8 + add rdi, 8 + loop .preserved + stmxcsr [actual_mxcsr] + mov eax, [original_mxcsr] + cmp eax, [actual_mxcsr] + jne fail + + ; Leave the last 8 bytes accessible, followed by a protected page. + mov eax, 9 + xor edi, edi + mov esi, 131072 + mov edx, 3 + mov r10d, 0x22 + mov r8, -1 + xor r9d, r9d + syscall + cmp rax, -4095 + jae fail + mov r12, rax + lea r13, [rax + 65536] + mov rax, 0x7ff0000000000001 + mov [r13 - 8], rax + mov rax, 0xfedcba9876543210 + mov [r13], rax + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + xor edx, edx + syscall + test eax, eax + jne fail + + lea rdi, [r13 - 8] + vbroadcastsd ymm0, [rdi] + CHECK + + ; A genuine cross-page fault must be restartable with the right address. + mov eax, 13 + mov edi, 11 + lea rsi, [action] + xor edx, edx + mov r10d, 8 + syscall + test eax, eax + jne fail + lea rdi, [r13 - 7] + vbroadcastsd ymm0, [rdi] + CHECK + cmp dword [faults], 1 + jne fail + mov eax, 11 + mov rdi, r12 + mov esi, 131072 + syscall + xor edi, edi + jmp exit + +handler: + cmp edi, 11 + jne fail + cmp [rsi + 16], r13 + jne fail + inc dword [faults] + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + mov edx, 3 + syscall + test eax, eax + jne fail + ret +restorer: + mov eax, 15 + syscall +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +sentinel: dq 0x0123456789abcdef, 0xfedcba9876543210 + dq 0x7ff0000000000001, 0x8000000000000000 +data: times 16 dq 0x7ff0000000000001, 0x8000000000000000 +action: dq handler, 0x04000004, restorer, 0 + +section .bss align=32 +actual: resb 32 +original_mxcsr: resd 1 +actual_mxcsr: resd 1 +faults: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vcmpsd-writeback.asm b/tests/integration/vcmpsd-writeback.asm new file mode 100644 index 0000000000..90a4a76667 --- /dev/null +++ b/tests/integration/vcmpsd-writeback.asm @@ -0,0 +1,171 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%define LESS 0x96969696 +%define EQUAL 0xa5a5a5a5 +%define GREATER 0xf0f0f0f0 +%define UNORDERED 0x87788778 + +%macro VERIFY 2 + vmovdqu [output], %1 + bt %2, (imm & 31) + sbb rdx, rdx + cmp [output], rdx + jne fail + mov rax, [r8 + 8] + cmp [output + 8], rax + jne fail + mov rax, [output + 16] + or rax, [output + 24] + jne fail +%endmacro + +%macro COMPARE 3 + ldmxcsr [masked] + vmovdqu ymm0, [r8] + vmovdqu ymm1, [r8] + vmovdqu ymm2, [r9] + vcmpsd %1, %2, %3, imm + stmxcsr [actual_mxcsr] +%endmacro + +%macro SAME_MXCSR 0 + mov eax, [actual_mxcsr] + cmp eax, [reference_mxcsr] + jne fail +%endmacro + +%macro SAVE_MXCSR 0 + mov eax, [actual_mxcsr] + mov [reference_mxcsr], eax +%endmacro + +%macro CHECK_FINITE_MXCSR 0 + ; A high-lane SNaN must not raise invalid for finite low lanes. + cmp r11d, UNORDERED + je %%unordered + cmp eax, [masked] + jne fail +%%unordered: +%endmacro + +section .text +_start: + ; Put memory operands at the last eight bytes before an inaccessible page. + xor edi, edi + mov esi, 8192 + mov edx, 3 + mov r10d, 0x22 + mov r8, -1 + xor r9d, r9d + mov eax, 9 + syscall + test rax, rax + js fail + lea r13, [rax + 4088] + lea rdi, [rax + 4096] + mov esi, 4096 + xor edx, edx + mov eax, 10 + syscall + test eax, eax + jnz fail + lea r14, [pairs] + lea r15, [pairs_end] +.pair: + mov r8, [r14] + mov r9, [r14 + 8] + mov r11d, [r14 + 16] + mov r12d, [r14 + 20] + mov rax, [r9] + mov [r13], rax + %assign imm 0 + %rep 256 + ; Aliasing src1 retains the original temporary-result path. + COMPARE xmm0, xmm0, xmm2 + VERIFY ymm0, r11d + SAVE_MXCSR + CHECK_FINITE_MXCSR + COMPARE xmm0, xmm1, xmm2 + VERIFY ymm0, r11d + SAME_MXCSR + COMPARE xmm2, xmm1, xmm2 + VERIFY ymm2, r11d + SAME_MXCSR + COMPARE xmm0, xmm0, [r13] + VERIFY ymm0, r11d + SAME_MXCSR + COMPARE xmm0, xmm1, [r13] + VERIFY ymm0, r11d + SAME_MXCSR + + COMPARE xmm1, xmm1, xmm1 + VERIFY ymm1, r12d + SAVE_MXCSR + COMPARE xmm0, xmm1, xmm1 + VERIFY ymm0, r12d + SAME_MXCSR + + ldmxcsr [masked] + vmovdqu ymm8, [r8] + vmovdqu ymm9, [r8] + vmovdqu ymm10, [r9] + vcmpsd xmm8, xmm9, xmm10, imm + VERIFY ymm8, r11d + %assign imm imm + 1 + %endrep + add r14, 24 + cmp r14, r15 + jb .pair + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +one: dq 1.0, 0x7ff0000000000123, -1, -1 +two: dq 2.0, 0x123456789abcdef0, -1, -1 +qnan: dq 0x7ff8000000000001, 0x7ff0000000000456, -1, -1 +snan: dq 0x7ff0000000000001, 0xfedcba9876543210, -1, -1 +zero: dq 0, 0x7ff0000000000123, -1, -1 +negative_zero: dq 0x8000000000000000, 0x0123456789abcdef, -1, -1 +infinity: dq 0x7ff0000000000000, 0x7ff0000000000123, -1, -1 +negative_infinity: dq 0xfff0000000000000, 0x7ff0000000000123, -1, -1 +largest: dq 0x7fefffffffffffff, 0x7ff0000000000123, -1, -1 +pairs: + dq one, two + dd LESS, EQUAL + dq one, one + dd EQUAL, EQUAL + dq two, one + dd GREATER, EQUAL + dq qnan, one + dd UNORDERED, UNORDERED + dq one, snan + dd UNORDERED, EQUAL + dq snan, qnan + dd UNORDERED, UNORDERED + dq zero, negative_zero + dd EQUAL, EQUAL + dq negative_infinity, one + dd LESS, EQUAL + dq infinity, largest + dd GREATER, EQUAL + dq infinity, infinity + dd EQUAL, EQUAL +pairs_end: +masked: dd 0x1f80 + +section .bss align=32 +output: resb 32 +actual_mxcsr: resd 1 +reference_mxcsr: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vcomis-flag-reduction.asm b/tests/integration/vcomis-flag-reduction.asm new file mode 100644 index 0000000000..0c241c0f2a --- /dev/null +++ b/tests/integration/vcomis-flag-reduction.asm @@ -0,0 +1,93 @@ +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +; Each subset is consumed before XOR makes all remaining flags dead. +%macro CHECK 5 + vmovdqu ymm0, [%2] + vmovdqu ymm1, [%3] + %1 xmm0, xmm1 +%if %5 & 1 + setc [result] +%endif +%if %5 & 2 + setp [result + 1] +%endif +%if %5 & 4 + lahf + mov [result + 2], ah +%endif +%if %5 & 8 + setz [result + 3] +%endif +%if %5 & 16 + sets [result + 4] +%endif +%if %5 & 32 + seto [result + 5] +%endif + xor eax, eax +%if %5 & 1 + cmp byte [result], (%4 & 1) != 0 + jne fail +%endif +%if %5 & 2 + cmp byte [result + 1], (%4 & 4) != 0 + jne fail +%endif +%if %5 & 4 + cmp byte [result + 2], %4 | 2 + jne fail +%endif +%if %5 & 8 + cmp byte [result + 3], (%4 & 0x40) != 0 + jne fail +%endif +%if %5 & 16 + cmp byte [result + 4], 0 + jne fail +%endif +%if %5 & 32 + cmp byte [result + 5], 0 + jne fail +%endif +%endmacro + +%macro SUITE 2 +%assign mask 0 +%rep 64 + CHECK %1, two_%2, one_%2, 0, mask + CHECK %1, one_%2, two_%2, 1, mask + CHECK %1, one_%2, one_%2, 0x40, mask + CHECK %1, nan_%2, one_%2, 0x45, mask +%assign mask mask + 1 +%endrep +%endmacro + +section .text +_start: + ldmxcsr [clean] + SUITE vcomiss, s + SUITE vucomiss, s + SUITE vcomisd, d + SUITE vucomisd, d + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +one_s: dd 0x3f800000, 1, 2, 3, 4, 5, 6, 7 +two_s: dd 0x40000000, 8, 9, 10, 11, 12, 13, 14 +nan_s: dd 0x7fc00001, 15, 16, 17, 18, 19, 20, 21 +one_d: dq 0x3ff0000000000000, 22, 23, 24 +two_d: dq 0x4000000000000000, 25, 26, 27 +nan_d: dq 0x7ff8000000000001, 28, 29, 30 +clean: dd 0x1f80 +section .bss +result: resb 6 +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vcvtdq2pd-ymm-alias.asm b/tests/integration/vcvtdq2pd-ymm-alias.asm new file mode 100644 index 0000000000..04317cfbdb --- /dev/null +++ b/tests/integration/vcvtdq2pd-ymm-alias.asm @@ -0,0 +1,64 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +section .text + +%macro CHECK_RESULT 1 + vmovdqu [actual], ymm0 + mov rax, [actual] + cmp rax, [%1] + jne fail + mov rax, [actual + 8] + cmp rax, [%1 + 8] + jne fail + mov rax, [actual + 16] + cmp rax, [%1 + 16] + jne fail + mov rax, [actual + 24] + cmp rax, [%1 + 24] + jne fail +%endmacro + +_start: + vmovdqu xmm1, [input] + vpxor ymm0, ymm0, ymm0 + vcvtdq2pd ymm0, xmm1 + CHECK_RESULT expected + + vmovdqu ymm1, [alias_input] + vcvtdq2pd ymm1, xmm1 + vmovdqu ymm0, ymm1 + CHECK_RESULT expected + + vpxor ymm0, ymm0, ymm0 + vcvtdq2pd ymm0, [input] + CHECK_RESULT expected + + vmovdqu ymm2, [alias_input] + vcvtdq2pd ymm2, [input] + vmovdqu ymm0, ymm2 + CHECK_RESULT expected + + xor edi, edi +fail: + mov eax, 60 + syscall + +section .rodata align=32 +input: + dd 0, 0x80000000, 0x7fffffff, -1 + dd 1, 2, 3, 4 +alias_input: + dd 0, 0x80000000, 0x7fffffff, -1 + dq 0xdeadbeefdeadbeef, 0xdeadbeefdeadbeef +expected: + dq 0x0000000000000000, 0xc1e0000000000000 + dq 0x41dfffffffc00000, 0xbff0000000000000 + +section .bss align=32 +actual: resb 32 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vcvtpd-register-alias.asm b/tests/integration/vcvtpd-register-alias.asm new file mode 100644 index 0000000000..947bbb95a1 --- /dev/null +++ b/tests/integration/vcvtpd-register-alias.asm @@ -0,0 +1,81 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +BITS 64 +default rel +global _start +section .text + +%macro CONVERT 4 + vmovdqu ymm0, [poison] + vmovdqu ymm1, [input] +%if %4 = 0 + vmovdqu ymm0, [input] +%if %3 = 256 + %1 xmm0, ymm0 +%else + %1 xmm0, xmm0 +%endif +%elif %4 = 1 +%if %3 = 256 + %1 xmm0, ymm1 +%else + %1 xmm0, xmm1 +%endif +%else +%if %3 = 256 + %1 xmm0, yword [input] +%else + %1 xmm0, oword [input] +%endif +%endif + vmovdqu [output], ymm0 +%if %2 + mov rax, 0x400000003f800000 +%else + mov rax, 0x0000000200000001 +%endif + cmp [output], rax + jne fail +%if %3 = 128 + xor eax, eax +%elif %2 + mov rax, 0x4080000040400000 +%else + mov rax, 0x0000000400000003 +%endif + cmp [output + 8], rax + jne fail + cmp qword [output + 16], 0 + jne fail + cmp qword [output + 24], 0 + jne fail +%endmacro + +_start: +; Repeat translations so a broken release build flushes its buffered diagnostic. +%rep 32 +%assign kind 0 +%rep 3 + CONVERT vcvtpd2ps, 1, 128, kind + CONVERT vcvtpd2ps, 1, 256, kind + CONVERT vcvtpd2dq, 0, 128, kind + CONVERT vcvtpd2dq, 0, 256, kind + CONVERT vcvttpd2dq, 0, 128, kind + CONVERT vcvttpd2dq, 0, 256, kind +%assign kind kind + 1 +%endrep +%endrep + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +input: dq 1.0, 2.0, 3.0, 4.0 +poison: times 4 dq -1 +section .bss align=32 +output: resb 32 +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vcvtsi2sd-alias.asm b/tests/integration/vcvtsi2sd-alias.asm new file mode 100644 index 0000000000..d5d48ff847 --- /dev/null +++ b/tests/integration/vcvtsi2sd-alias.asm @@ -0,0 +1,94 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro CHECK_RESULT 1 + vmovdqu [actual], ymm0 + mov rax, [actual] + mov rdx, %1 + cmp rax, rdx + jne fail + mov rax, [actual + 8] + cmp rax, [initial + 8] + jne fail + mov rax, [actual + 16] + or rax, [actual + 24] + jne fail +%endmacro + +%macro PREPARE 1 + vmovdqu ymm0, [initial] + vmovdqu ymm1, [initial] +%if %1 + vmovdqu ymm2, [initial] +%endif +%endmacro + +_start: + PREPARE 1 + mov eax, [src32] + vcvtsi2sd xmm0, xmm0, eax + CHECK_RESULT 0xc1e0000000000000 + + PREPARE 1 + mov eax, [src32 + 4] + vcvtsi2sd xmm0, xmm0, eax + CHECK_RESULT 0x41dfffffffc00000 + + PREPARE 1 + mov eax, [src32 + 8] + vcvtsi2sd xmm0, xmm0, eax + CHECK_RESULT 0xbff0000000000000 + + PREPARE 1 + mov rax, [src64] + vcvtsi2sd xmm0, xmm0, rax + CHECK_RESULT 0x4270000000001000 + + PREPARE 1 + mov rax, [src64 + 8] + vcvtsi2sd xmm0, xmm0, rax + CHECK_RESULT 0xc270000000001000 + + PREPARE 1 + mov eax, [src32] + vcvtsi2sd xmm0, xmm1, eax + CHECK_RESULT 0xc1e0000000000000 + + PREPARE 1 + mov rax, [src64] + vcvtsi2sd xmm0, xmm1, rax + CHECK_RESULT 0x4270000000001000 + + PREPARE 0 + vcvtsi2sd xmm0, xmm0, dword [src32 + 4] + CHECK_RESULT 0x41dfffffffc00000 + + PREPARE 0 + vcvtsi2sd xmm0, xmm0, qword [src64 + 8] + CHECK_RESULT 0xc270000000001000 + + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +initial: + dq 0x0123456789abcdef, 0x1122334455667788 + dq 0xdeadbeefdeadbeef, 0xfedcba9876543210 +src32: + dd 0x80000000, 0x7fffffff, -1 +src64: + dq 0x0000010000000001, -0x0000010000000001 + +section .bss align=32 +actual: resb 32 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vcvtss2sd-alias.asm b/tests/integration/vcvtss2sd-alias.asm new file mode 100644 index 0000000000..5405112175 --- /dev/null +++ b/tests/integration/vcvtss2sd-alias.asm @@ -0,0 +1,110 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro VERIFY 1 + vmovdqu [output], %1 + mov rax, [output] + cmp rax, [r10 + 8] + jne fail + mov rax, [output + 8] + cmp rax, [initial + 8] + jne fail + mov rax, [output + 16] + or rax, [output + 24] + jne fail +%endmacro + +%macro TEST 0 + vmovdqu ymm0, [initial] + vmovdqu ymm1, [initial] + vmovdqu ymm2, [initial] + vcvtss2sd xmm0, xmm1, xmm2 + VERIFY ymm0 + vmovdqu ymm1, [initial] + vcvtss2sd xmm1, xmm1, xmm2 + VERIFY ymm1 + vmovdqu ymm2, [initial] + vcvtss2sd xmm2, xmm1, xmm2 + VERIFY ymm2 + vmovdqu ymm1, [initial] + vcvtss2sd xmm0, xmm1, xmm1 + VERIFY ymm0 + vcvtss2sd xmm1, xmm1, xmm1 + VERIFY ymm1 + vmovdqu ymm1, [initial] + vcvtss2sd xmm0, xmm1, [initial] + VERIFY ymm0 + vcvtss2sd xmm1, xmm1, [initial] + VERIFY ymm1 +%endmacro + +section .text +_start: + xor r9d, r9d +.rounding: + mov eax, r9d + shl eax, 13 + or eax, 0x1f80 + mov [mxcsr], eax + ldmxcsr [mxcsr] + lea r10, [cases] + mov r11d, 12 +.case: + mov eax, [r10] + mov [initial], eax + TEST + add r10, 16 + dec r11d + jnz .case + inc r9d + cmp r9d, 4 + jne .rounding + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=8 +cases: + dd 0, 0 + dq 0 + dd 0x80000000, 0 + dq 0x8000000000000000 + dd 0x3f000000, 0 + dq 0x3fe0000000000000 + dd 0xbf000000, 0 + dq 0xbfe0000000000000 + dd 0x3fc00000, 0 + dq 0x3ff8000000000000 + dd 0xbfc00000, 0 + dq 0xbff8000000000000 + dd 0x00800000, 0 + dq 0x3810000000000000 + dd 0x80800000, 0 + dq 0xb810000000000000 + dd 0x7f7fffff, 0 + dq 0x47efffffe0000000 + dd 0xff7fffff, 0 + dq 0xc7efffffe0000000 + dd 0x7f800000, 0 + dq 0x7ff0000000000000 + dd 0xff800000, 0 + dq 0xfff0000000000000 + +section .data align=32 +initial: + dd 0, 0x89abcdef + dq 0x123456789abcdef0, -1, -1 + +section .bss align=32 +output: resb 32 +mxcsr: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vinsertf128-alias.asm b/tests/integration/vinsertf128-alias.asm new file mode 100644 index 0000000000..533f3b8475 --- /dev/null +++ b/tests/integration/vinsertf128-alias.asm @@ -0,0 +1,94 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +section .text + +%macro CHECK 2 + vmovdqu [actual], ymm0 + mov edi, %2 + mov rax, [%1] + cmp rax, [actual] + jne fail + mov rax, [%1 + 8] + cmp rax, [actual + 8] + jne fail + mov rax, [%1 + 16] + cmp rax, [actual + 16] + jne fail + mov rax, [%1 + 24] + cmp rax, [actual + 24] + jne fail +%endmacro + +%macro ALIASES 1 + lea rbx, [expected_ab_low] + lea rdx, [expected_aa_low] + test byte [immediate], 1 + jz %%expected + lea rbx, [expected_ab_high] + lea rdx, [expected_aa_high] +%%expected: + vmovdqu ymm0, [junk] + vmovdqu ymm1, [input_a] + vmovdqu ymm2, [input_b] + vinsertf128 ymm0, ymm1, xmm2, %1 + CHECK rbx, 1 + vmovdqu ymm0, [input_a] + vinsertf128 ymm0, ymm0, xmm2, %1 + CHECK rbx, 2 + vmovdqu ymm0, [input_b] + vinsertf128 ymm0, ymm1, xmm0, %1 + CHECK rbx, 3 + vinsertf128 ymm0, ymm1, xmm1, %1 + CHECK rdx, 4 + vmovdqu ymm0, [input_a] + vinsertf128 ymm0, ymm0, xmm0, %1 + CHECK rdx, 5 + vinsertf128 ymm0, ymm1, [input_b], %1 + CHECK rbx, 6 + vmovdqu ymm0, [input_a] + vinsertf128 ymm0, ymm0, [input_b], %1 + CHECK rbx, 7 +%endmacro + +_start: +%assign imm 0 +%rep 256 + mov byte [immediate], imm + ALIASES imm +%assign imm imm+1 +%endrep + ; A pending VEX upper-half clear must be applied before reading src1. + vmovdqu ymm0, [input_a] + vmovdqu ymm2, [input_b] + vpxor xmm0, xmm0, xmm0 + vinsertf128 ymm0, ymm0, xmm2, 0 + CHECK expected_pending, 8 + xor edi, edi +fail: + mov eax, 60 + syscall + +section .rodata align=32 +input_a: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0x123456789abcdef0, 0xfedcba9876543210 +input_b: dq 0xfff800000000cafe, 0x0123456789abcdef + dq 0xaaaaaaaa55555555, 0x9999999966666666 +junk: times 4 dq 0xdeadbeefdeadbeef +expected_ab_low: dq 0xfff800000000cafe, 0x0123456789abcdef + dq 0x123456789abcdef0, 0xfedcba9876543210 +expected_ab_high: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0xfff800000000cafe, 0x0123456789abcdef +expected_aa_low: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0x123456789abcdef0, 0xfedcba9876543210 +expected_aa_high: times 2 dq 0x7ff0000000000001, 0x8000000000000000 +expected_pending: dq 0xfff800000000cafe, 0x0123456789abcdef, 0, 0 + +section .bss align=32 +actual: resb 32 +immediate: resb 1 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vinsertf128-high-pending.asm b/tests/integration/vinsertf128-high-pending.asm new file mode 100644 index 0000000000..99a756caab --- /dev/null +++ b/tests/integration/vinsertf128-high-pending.asm @@ -0,0 +1,229 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +%macro CHECK 2 + vmovdqu [actual], %1 + lea rsi, [actual] + lea rdi, [%2] + mov ecx, 4 +%%next: + mov rax, [rsi] + cmp rax, [rdi] + jne fail + add rsi, 8 + add rdi, 8 + loop %%next +%endmacro + +%macro INSERT 3 + vmovdqu ymm0, [input_b] + vmovdqu ymm1, [input_a] + vmovdqu ymm2, [input_b] + vmovdqa xmm1, xmm1 + vmovdqa xmm0, xmm0 + %if %2 = 0 + %3 ymm0, ymm1, xmm2, %1 + %elif %2 = 1 + vmovdqa xmm0, xmm1 + %3 ymm0, ymm0, xmm2, %1 + %elif %2 = 2 + %3 ymm0, ymm1, xmm0, %1 + %elif %2 = 3 + %3 ymm0, ymm1, xmm1, %1 + %elif %2 = 4 + vmovdqa xmm0, xmm1 + %3 ymm0, ymm0, xmm0, %1 + %elif %2 = 5 + %3 ymm0, ymm1, [input_b], %1 + %else + vmovdqa xmm0, xmm1 + %3 ymm0, ymm0, [input_b], %1 + %endif + %if %2 = 3 || %2 = 4 + %if %1 & 1 + CHECK ymm0, expected_aa_high + %else + CHECK ymm0, expected_a_zero + %endif + %else + %if %1 & 1 + CHECK ymm0, expected_ab_high + %else + CHECK ymm0, expected_b_zero + %endif + %endif + CHECK ymm1, expected_a_zero +%endmacro + +; %1 selects distinct registers or dest == src1. Read exactly 16 bytes. +%macro MEMORY 3 + vmovdqu ymm0, [input_b] + vmovdqu ymm1, [input_a] + lea rdi, [r13 - %2] + vmovdqa xmm0, xmm0 + vmovdqa xmm1, xmm1 + %if %1 = 1 + %3 ymm0, ymm0, [rdi], 255 + CHECK ymm0, expected_bb_high + %else + %3 ymm0, ymm1, [rdi], 255 + CHECK ymm0, expected_ab_high + %endif + CHECK ymm1, expected_a_zero +%endmacro + +section .text +_start: + stmxcsr [original_mxcsr] +%assign imm 0 +%rep 256 +%assign alias 0 +%rep 7 + INSERT imm, alias, vinsertf128 + INSERT imm, alias, vinserti128 +%assign alias alias + 1 +%endrep +%assign imm imm + 1 +%endrep + stmxcsr [actual_mxcsr] + mov eax, [original_mxcsr] + cmp eax, [actual_mxcsr] + jne fail + + mov eax, 9 + xor edi, edi + mov esi, 131072 + mov edx, 3 + mov r10d, 0x22 + mov r8, -1 + xor r9d, r9d + syscall + cmp rax, -4095 + jae fail + mov r12, rax + lea r13, [rax + 65536] + mov rax, [input_b] + mov [r13 - 16], rax + mov rax, [input_b + 8] + mov [r13 - 8], rax + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + xor edx, edx + syscall + test eax, eax + jne fail + MEMORY 0, 16, vinsertf128 + MEMORY 1, 16, vinsertf128 + MEMORY 0, 16, vinserti128 + MEMORY 1, 16, vinserti128 + + mov eax, 13 + mov edi, 11 + lea rsi, [action] + xor edx, edx + mov r10d, 8 + syscall + test eax, eax + jne fail +%macro CROSS_PAGE 2 + ; Reprotect before each cross-page load, then recreate deferred clears. + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + mov edx, 3 + syscall + mov rax, [input_b] + mov [r13 - 15], rax + mov rax, [input_b + 8] + mov [r13 - 7], rax + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + xor edx, edx + syscall + test eax, eax + jne fail + MEMORY %1, 15, %2 +%endmacro + CROSS_PAGE 0, vinsertf128 + CROSS_PAGE 1, vinsertf128 + CROSS_PAGE 0, vinserti128 + CROSS_PAGE 1, vinserti128 + cmp dword [faults], 4 + jne fail + mov eax, 11 + mov rdi, r12 + mov esi, 131072 + syscall + xor edi, edi + jmp exit + +handler: + cmp edi, 11 + jne fail + cmp [rsi + 16], r13 + jne fail + ; x86-64 ucontext holds fpregs at 224; XSAVE YMM_Hi128 starts at 576. + mov r8, [rdx + 224] + mov rax, [input_b] + cmp [r8 + 160], rax + jne fail + mov rax, [input_b + 8] + cmp [r8 + 168], rax + jne fail + mov rax, [input_a] + cmp [r8 + 176], rax + jne fail + mov rax, [input_a + 8] + cmp [r8 + 184], rax + jne fail + cmp qword [r8 + 576], 0 + jne fail + cmp qword [r8 + 584], 0 + jne fail + cmp qword [r8 + 592], 0 + jne fail + cmp qword [r8 + 600], 0 + jne fail + inc dword [faults] + mov eax, 10 + mov rdi, r13 + mov esi, 65536 + mov edx, 3 + syscall + test eax, eax + jne fail + ret +restorer: + mov eax, 15 + syscall +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +input_a: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0x123456789abcdef0, 0xfedcba9876543210 +input_b: dq 0xfff800000000cafe, 0x0123456789abcdef + dq 0xaaaaaaaa55555555, 0x9999999966666666 +expected_a_zero: dq 0x7ff0000000000001, 0x8000000000000000, 0, 0 +expected_b_zero: dq 0xfff800000000cafe, 0x0123456789abcdef, 0, 0 +expected_ab_high: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0xfff800000000cafe, 0x0123456789abcdef +expected_aa_high: times 2 dq 0x7ff0000000000001, 0x8000000000000000 +expected_bb_high: times 2 dq 0xfff800000000cafe, 0x0123456789abcdef +action: dq handler, 0x04000004, restorer, 0 + +section .bss align=32 +actual: resb 32 +original_mxcsr: resd 1 +actual_mxcsr: resd 1 +faults: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vmovd-xmm-zero.asm b/tests/integration/vmovd-xmm-zero.asm new file mode 100644 index 0000000000..42163a06e0 --- /dev/null +++ b/tests/integration/vmovd-xmm-zero.asm @@ -0,0 +1,98 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro VERIFY 2 + vmovdqu [actual], ymm%1 + mov eax, [actual] + cmp eax, %2 + jne fail + mov rax, [actual + 4] + or rax, [actual + 12] + or rax, [actual + 20] + or eax, [actual + 28] + jne fail +%endmacro + +%macro GPR_TEST 1 + vmovdqu ymm%1, [junk] + vxorps xmm%1, xmm%1, xmm%1 + mov rax, -1 + mov eax, 0x89abcdef + vmovd xmm%1, eax + call %%next +%%next: + add rsp, 8 + VERIFY %1, 0x89abcdef +%endmacro + +%macro MEMORY_TEST 1 + vmovdqu ymm%1, [junk] + vmovss xmm%1, xmm%1, xmm%1 + mov dword [input + 1], 0xfedcba98 + vmovd xmm%1, [input + 1] + call %%next +%%next: + add rsp, 8 + VERIFY %1, 0xfedcba98 +%endmacro + +section .text +_start: + %assign reg 0 + %rep 16 + GPR_TEST reg + MEMORY_TEST reg + %assign reg reg + 1 + %endrep + + ; A four-byte load at a page end must not read the protected next page. + mov eax, 9 + xor edi, edi + mov esi, 8192 + mov edx, 3 + mov r10d, 0x22 + mov r8, -1 + xor r9d, r9d + syscall + cmp rax, -4095 + jae fail + mov r12, rax + mov dword [r12 + 4092], 0xfedcba98 + lea rdi, [r12 + 4096] + mov esi, 4096 + xor edx, edx + mov eax, 10 + syscall + test eax, eax + jne fail + vmovdqu ymm0, [junk] + vmovd xmm0, [r12 + 4092] + VERIFY 0, 0xfedcba98 + mov eax, 11 + mov rdi, r12 + mov esi, 8192 + syscall + test eax, eax + jne fail + + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +junk: times 4 dq 0xdeadbeefdeadbeef + +section .bss align=32 +actual: resb 32 +padding: resb 1 +input: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vmovsd-alias.asm b/tests/integration/vmovsd-alias.asm new file mode 100644 index 0000000000..008a97ab83 --- /dev/null +++ b/tests/integration/vmovsd-alias.asm @@ -0,0 +1,109 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +section .text + +%macro CASE 4 + vmovdqu ymm%1, [pattern_a] + vmovdqu ymm%2, [pattern_b] + vmovdqu ymm%3, [pattern_c] +%if %4 + vxorps xmm%2, xmm%2, xmm%2 +%endif + vmovdqu [saved1], ymm%2 + vmovdqu [saved2], ymm%3 + mov rax, [saved2] + mov [expected], rax + mov rax, [saved1 + 8] + mov [expected + 8], rax + mov qword [expected + 16], 0 + mov qword [expected + 24], 0 +%if %4 + vxorps xmm%2, xmm%2, xmm%2 +%endif + stmxcsr [mxcsr_before] + vmovsd xmm%1, xmm%2, xmm%3 + vmovdqu [actual], ymm%1 + stmxcsr [mxcsr_after] + inc edi + mov eax, [mxcsr_before] + cmp eax, [mxcsr_after] + jne fail +%assign lane 0 +%rep 4 + mov rax, [actual + lane * 8] + cmp rax, [expected + lane * 8] + jne fail +%assign lane lane + 1 +%endrep +%if %2 != %1 + vmovdqu [actual], ymm%2 +%assign lane 0 +%rep 4 + mov rax, [actual + lane * 8] + cmp rax, [saved1 + lane * 8] + jne fail +%assign lane lane + 1 +%endrep +%endif +%if %3 != %1 + vmovdqu [actual], ymm%3 +%assign lane 0 +%rep 4 + mov rax, [actual + lane * 8] + cmp rax, [saved2 + lane * 8] + jne fail +%assign lane lane + 1 +%endrep +%endif +%endmacro + +_start: + xor edi, edi +%assign rc 0 +%rep 4 + mov dword [mxcsr_before], 0x1f80 | (rc << 13) + ldmxcsr [mxcsr_before] +%assign d 0 +%rep 16 +%assign s1 (d + 1) % 16 +%assign s2 (d + 2) % 16 + CASE d, s1, s2, 0 + CASE d, d, s2, 0 + CASE d, s1, d, 0 + CASE d, s1, s1, 0 + CASE d, d, d, 0 + CASE d, s1, s2, 1 + CASE d, d, s2, 1 + CASE d, s1, d, 1 + CASE d, s1, s1, 1 + CASE d, d, d, 1 +%assign d d + 1 +%endrep +%assign rc rc + 1 +%endrep + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +pattern_a: dq 0x8000000000000000, 0x7ff0000000000001, -1, 0x123456789abcdef0 +pattern_b: dq 0x7ff8000000000055, 0x0123456789abcdef, 0x7777777777777777, -1 +pattern_c: dq 0x7ff0000000000001, 0x5555555555555555, -1, -1 + +section .bss align=32 +actual: resb 32 +expected: resb 32 +saved1: resb 32 +saved2: resb 32 +mxcsr_before: resd 1 +mxcsr_after: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vmovss-memory-zero.asm b/tests/integration/vmovss-memory-zero.asm new file mode 100644 index 0000000000..3d8e781712 --- /dev/null +++ b/tests/integration/vmovss-memory-zero.asm @@ -0,0 +1,117 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro VERIFY 1 + vmovdqu [output], ymm%1 + mov eax, [output] + cmp eax, [input] + jne fail + cmp dword [output + 4], 0 + jne fail + mov rax, [output + 8] + or rax, [output + 16] + or rax, [output + 24] + jne fail + stmxcsr [status] + mov eax, [status] + cmp eax, [control] + jne fail +%endmacro + +%macro TEST 1 + vmovdqu ymm%1, [initial] + vmovss xmm%1, [input] + VERIFY %1 + ; Overwrite an older deferred upper-half clear before a TB boundary. + vmovdqu ymm%1, [initial] + vmovss xmm%1, xmm%1, xmm%1 + vmovss xmm%1, [input] + call %%next +%%next: + add rsp, 8 + VERIFY %1 +%endmacro + +section .text +_start: + xor r9d, r9d +.rounding: + mov eax, r9d + shl eax, 13 + or eax, 0x1fbf + mov [control], eax + ldmxcsr [control] + lea r10, [values] + mov r11d, 16 +.value: + mov eax, [r10] + mov [input], eax + %assign reg 0 + %rep 16 + TEST reg + %assign reg reg + 1 + %endrep + add r10, 4 + dec r11d + jnz .value + inc r9d + cmp r9d, 4 + jne .rounding + ; A four-byte load at a page end must not read the protected next page. + mov eax, 9 + xor edi, edi + mov esi, 8192 + mov edx, 3 + mov r10d, 0x22 + mov r8, -1 + xor r9d, r9d + syscall + cmp rax, -4095 + jae fail + mov r12, rax + mov dword [r12 + 4092], 0x7f800001 + lea rdi, [r12 + 4096] + mov esi, 4096 + xor edx, edx + mov eax, 10 + syscall + test eax, eax + jne fail + vmovdqu ymm0, [initial] + vmovss xmm0, [r12 + 4092] + mov dword [input], 0x7f800001 + VERIFY 0 + mov eax, 11 + mov rdi, r12 + mov esi, 8192 + syscall + test eax, eax + jne fail + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +initial: dq -1, 0x123456789abcdef0, -1, 0xfedcba9876543210 +values: + dd 0, 0x80000000, 1, 0x80000001 + dd 0x007fffff, 0x807fffff, 0x00800000, 0x80800000 + dd 0x3f800000, 0xbf800000, 0x7f7fffff, 0xff7fffff + dd 0x7f800000, 0xff800000, 0x7fc00001, 0x7f800001 + +section .bss align=32 +output: resb 32 +control: resd 1 +status: resd 1 +padding: resb 1 +input: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vperm2x128-immediate.asm b/tests/integration/vperm2x128-immediate.asm new file mode 100644 index 0000000000..9f1442ba67 --- /dev/null +++ b/tests/integration/vperm2x128-immediate.asm @@ -0,0 +1,98 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +section .text + +%macro CASE 3 + vmovdqu ymm0, [junk] + vmovdqu ymm1, [input_a] + vmovdqu ymm2, [input_b] + %assign source_low 0 + %assign source_high 0 + %if %3 == 0 + %1 ymm0, ymm1, ymm2, %2 + %elif %3 == 1 + %1 ymm1, ymm1, ymm2, %2 + vmovdqu ymm0, ymm1 + %elif %3 == 2 + %1 ymm2, ymm1, ymm2, %2 + vmovdqu ymm0, ymm2 + %elif %3 == 3 + %assign source_low 2 + %assign source_high 2 + %1 ymm0, ymm1, ymm1, %2 + %elif %3 == 4 + %assign source_low 2 + %assign source_high 2 + %1 ymm1, ymm1, ymm1, %2 + vmovdqu ymm0, ymm1 + %elif %3 == 5 + %1 ymm0, ymm1, [input_b], %2 + %elif %3 == 6 + %1 ymm1, ymm1, [input_b], %2 + vmovdqu ymm0, ymm1 + %else + ; The low half is read after a VEX.128 write made upper zero pending. + %assign source_low 2 + %assign source_high 2 + vmovdqu xmm1, [input_a] + %1 ymm0, ymm1, ymm1, %2 + %endif + vmovdqu [actual], ymm0 + %assign half 0 + %rep 2 + %assign select ((%2 >> (half * 4)) & 3) + %assign clear ((%2 >> (half * 4 + 3)) & 1) + %if half == 0 + %assign same source_low + %else + %assign same source_high + %endif + %assign lane 0 + %rep 2 + %if clear || (%3 == 7 && (select & 1)) + xor eax, eax + %elif select < 2 || same + mov rax, [input_a + (select & 1) * 16 + lane * 8] + %else + mov rax, [input_b + (select & 1) * 16 + lane * 8] + %endif + cmp [actual + half * 16 + lane * 8], rax + jne fail + %assign lane lane + 1 + %endrep + %assign half half + 1 + %endrep +%endmacro + +_start: + mov edi, 1 + %assign immediate 0 + %rep 256 + %assign alias 0 + %rep 8 + CASE vperm2f128, immediate, alias + CASE vperm2i128, immediate, alias + %assign alias alias + 1 + %endrep + %assign immediate immediate + 1 + %endrep + xor edi, edi +fail: + mov eax, 60 + syscall + +section .rodata align=32 +input_a: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0x0123456789abcdef, 0x1020304050607080 +input_b: dq 0x7ff8123456789abc, 0xffffffffffffffff + dq 0xfedcba9876543210, 0x8877665544332211 +junk: times 4 dq 0xdeadbeefdeadbeef + +section .bss align=32 +actual: resb 32 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vpermilpd-immediate.asm b/tests/integration/vpermilpd-immediate.asm new file mode 100644 index 0000000000..bc7ddff26c --- /dev/null +++ b/tests/integration/vpermilpd-immediate.asm @@ -0,0 +1,73 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro VERIFY 1 + vmovdqu [output], %1 + mov rax, [output] + cmp rax, [input + (imm & 1) * 8] + jne fail + mov rax, [output + 8] + cmp rax, [input + ((imm >> 1) & 1) * 8] + jne fail + mov rax, [output + 16] + or rax, [output + 24] + jne fail +%endmacro + +section .text +_start: + mov ecx, 4 +.rounding: + mov eax, ecx + dec eax + shl eax, 13 + or eax, 0x1fbf + mov [mxcsr], eax + ldmxcsr [mxcsr] + %assign imm 0 + %rep 256 + vmovdqu ymm0, [junk] + vmovdqu ymm1, [input] + vpermilpd xmm0, xmm1, imm + VERIFY ymm0 + vmovdqu ymm0, [input] + vpermilpd xmm0, xmm0, imm + VERIFY ymm0 + vmovdqu ymm0, [junk] + vpermilpd xmm0, [input], imm + VERIFY ymm0 + ; A pending source upper-zero must not affect the low-lane shuffle. + vmovdqu xmm1, [input] + vpermilpd xmm0, xmm1, imm + VERIFY ymm0 + %assign imm imm + 1 + %endrep + stmxcsr [actual_mxcsr] + mov eax, [actual_mxcsr] + cmp eax, [mxcsr] + jne fail + dec ecx + jnz .rounding + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +input: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0x0123456789abcdef, 0xfedcba9876543210 +junk: times 4 dq 0xdeadbeefdeadbeef + +section .bss align=32 +output: resb 32 +mxcsr: resd 1 +actual_mxcsr: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vpermilpd-ymm-immediate.asm b/tests/integration/vpermilpd-ymm-immediate.asm new file mode 100644 index 0000000000..dcc42eafdb --- /dev/null +++ b/tests/integration/vpermilpd-ymm-immediate.asm @@ -0,0 +1,75 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro VERIFY 2 + vmovdqu [output], %1 + %assign lane 0 + %rep 4 + mov rax, [output + lane * 8] + %if %2 && lane >= 2 + test rax, rax + %else + cmp rax, [input + ((lane & 2) + ((imm >> lane) & 1)) * 8] + %endif + jne fail + %assign lane lane + 1 + %endrep +%endmacro + +section .text +_start: + mov ecx, 4 +.rounding: + mov eax, ecx + dec eax + shl eax, 13 + or eax, 0x1fbf + mov [mxcsr], eax + ldmxcsr [mxcsr] + %assign imm 0 + %rep 256 + vmovdqu ymm0, [junk] + vmovdqu ymm1, [input] + vpermilpd ymm0, ymm1, imm + VERIFY ymm0, 0 + vmovdqu ymm0, [input] + vpermilpd ymm0, ymm0, imm + VERIFY ymm0, 0 + vmovdqu ymm0, [junk] + vpermilpd ymm0, [input], imm + VERIFY ymm0, 0 + vmovdqu ymm1, [junk] + vmovdqu xmm1, [input] + vpermilpd ymm0, ymm1, imm + VERIFY ymm0, 1 + %assign imm imm + 1 + %endrep + stmxcsr [actual_mxcsr] + mov eax, [actual_mxcsr] + cmp eax, [mxcsr] + jne fail + dec ecx + jnz .rounding + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=32 +input: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0x0123456789abcdef, 0xfedcba9876543210 +junk: times 4 dq 0xdeadbeefdeadbeef + +section .bss align=32 +output: resb 32 +mxcsr: resd 1 +actual_mxcsr: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vroundsd-fixed.asm b/tests/integration/vroundsd-fixed.asm new file mode 100644 index 0000000000..5552e8ff7f --- /dev/null +++ b/tests/integration/vroundsd-fixed.asm @@ -0,0 +1,128 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro VERIFY 2 + vmovdqu [output], %1 + mov rax, [output] + cmp rax, [rdi + %2 * 8] + jne fail + mov rax, [output + 8] + cmp rax, [rsi + 8] + jne fail + mov rax, [output + 16] + or rax, [output + 24] + jne fail +%endmacro + +%macro TEST 1 + %assign mode (%1 & 3) + %if %1 & 4 + mov rdi, r8 + %else + lea rdi, [r8 + mode * 8] + %endif + vmovdqu ymm0, [rsi] + vmovdqu ymm1, [rsi] + vmovdqu ymm2, [rsi] + vroundsd xmm0, xmm1, xmm2, %1 + VERIFY ymm0, 0 + vmovdqu ymm1, [rsi] + vroundsd xmm1, xmm1, xmm2, %1 + VERIFY ymm1, 0 + vmovdqu ymm1, [rsi] + vroundsd xmm2, xmm1, xmm2, %1 + VERIFY ymm2, 0 + vmovdqu ymm1, [rsi] + vroundsd xmm0, xmm1, xmm1, %1 + VERIFY ymm0, 0 + vroundsd xmm1, xmm1, xmm1, %1 + VERIFY ymm1, 0 + vmovdqu ymm1, [rsi] + vroundsd xmm0, xmm1, [rsi], %1 + VERIFY ymm0, 0 + vroundsd xmm1, xmm1, [rsi], %1 + VERIFY ymm1, 0 +%endmacro + +section .text +_start: + xor r9d, r9d +.rounding: + mov eax, r9d + shl eax, 13 + or eax, 0x1f80 + mov [mxcsr], eax + ldmxcsr [mxcsr] + lea r10, [cases] + mov r11d, 10 +.case: + lea rsi, [initial] + mov rax, [r10] + mov [rsi], rax + lea r8, [r10 + 8] + %assign imm 0 + %rep 16 + %if imm & 4 + lea r8, [r10 + 8 + r9 * 8] + %else + lea r8, [r10 + 8] + %endif + TEST imm + %assign imm imm + 1 + %endrep + add r10, 40 + dec r11d + jnz .case + inc r9d + cmp r9d, 4 + jne .rounding + ; An enabled x87 invalid trap must keep the original rounding path. + fldcw [control] + lea rsi, [initial] + mov rax, [cases + 4 * 40] + mov [rsi], rax + lea rdi, [cases + 4 * 40 + 8 + 3 * 8] + vmovdqu ymm1, [rsi] + vmovdqu ymm2, [rsi] + vroundsd xmm0, xmm1, xmm2, 3 + VERIFY ymm0, 0 + fninit + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata +align 8 +cases: + ; Input, nearest-even, floor, ceil, truncation. + dq 0, 0, 0, 0, 0 + dq 0x8000000000000000, 0x8000000000000000, 0x8000000000000000, 0x8000000000000000, 0x8000000000000000 + dq 0x3fe0000000000000, 0, 0, 0x3ff0000000000000, 0 + dq 0xbfe0000000000000, 0x8000000000000000, 0xbff0000000000000, 0x8000000000000000, 0x8000000000000000 + dq 0x3ff8000000000000, 0x4000000000000000, 0x3ff0000000000000, 0x4000000000000000, 0x3ff0000000000000 + dq 0xbff8000000000000, 0xc000000000000000, 0xc000000000000000, 0xbff0000000000000, 0xbff0000000000000 + dq 0x4004000000000000, 0x4000000000000000, 0x4000000000000000, 0x4008000000000000, 0x4000000000000000 + dq 0xc004000000000000, 0xc000000000000000, 0xc008000000000000, 0xc000000000000000, 0xc000000000000000 + dq 0x432fffffffffffff, 0x4330000000000000, 0x432ffffffffffffe, 0x4330000000000000, 0x432ffffffffffffe + dq 0x4340000000000000, 0x4340000000000000, 0x4340000000000000, 0x4340000000000000, 0x4340000000000000 +control: dw 0x37e + +section .data +align 32 +initial: + dq 0, 0x123456789abcdef0, -1, -1 + +section .bss +align 32 +output: resb 32 +mxcsr: resd 1 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vroundss-fixed.asm b/tests/integration/vroundss-fixed.asm new file mode 100644 index 0000000000..dab79e01a4 --- /dev/null +++ b/tests/integration/vroundss-fixed.asm @@ -0,0 +1,151 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only + +bits 64 +default rel +global _start + +%macro VERIFY 1 + vmovdqu [output], %1 + mov eax, [output] + cmp eax, [rdi] + jne fail + mov eax, [output + 4] + cmp eax, [initial + 4] + jne fail + mov rax, [output + 8] + cmp rax, [initial + 8] + jne fail + mov rax, [output + 16] + or rax, [output + 24] + jne fail +%endmacro + +%macro TEST 1 + %if %1 & 4 + lea rdi, [r10 + 4 + r9 * 4] + %else + lea rdi, [r10 + 4 + (%1 & 3) * 4] + %endif + vmovdqu ymm0, [initial] + vmovdqu ymm1, [initial] + vmovdqu ymm2, [initial] + vroundss xmm0, xmm1, xmm2, %1 + VERIFY ymm0 + vmovdqu ymm1, [initial] + vroundss xmm1, xmm1, xmm2, %1 + VERIFY ymm1 + vmovdqu ymm1, [initial] + vroundss xmm2, xmm1, xmm2, %1 + VERIFY ymm2 + vmovdqu ymm1, [initial] + vroundss xmm0, xmm1, xmm1, %1 + VERIFY ymm0 + vroundss xmm1, xmm1, xmm1, %1 + VERIFY ymm1 + vmovdqu ymm1, [initial] + vroundss xmm0, xmm1, [initial], %1 + VERIFY ymm0 + vroundss xmm1, xmm1, [initial], %1 + VERIFY ymm1 +%endmacro + +section .text +_start: + xor r9d, r9d +.rounding: + mov eax, r9d + shl eax, 13 + or eax, 0x1f80 + mov [mxcsr], eax + ldmxcsr [mxcsr] + lea r10, [cases] + mov r11d, 10 +.case: + mov eax, [r10] + mov [initial], eax + %assign imm 0 + %rep 16 + TEST imm + %assign imm imm + 1 + %endrep + add r10, 20 + dec r11d + jnz .case + inc r9d + cmp r9d, 4 + jne .rounding + ; A preexisting inexact flag must survive precision-suppressed rounds. + xor r9d, r9d +.sticky: + mov eax, r9d + shl eax, 13 + or eax, 0x1f80 + mov [mxcsr], eax + ldmxcsr [mxcsr] + lea r10, [cases + 4 * 20] + mov eax, [r10] + mov [initial], eax + fnclex + vmovss xmm15, [initial] + vroundss xmm15, xmm15, xmm15, 3 + fnstsw [status] + stmxcsr [mxcsr_before] + TEST 8 + TEST 9 + TEST 10 + TEST 11 + fnstsw ax + xor ax, [status] + test ax, 0x3f + jnz fail + stmxcsr [mxcsr] + mov eax, [mxcsr] + cmp eax, [mxcsr_before] + jne fail + inc r9d + cmp r9d, 4 + jne .sticky + ; Enabled x87 invalid must retain the original fixed-round path. + fldcw [control] + lea r10, [cases + 4 * 20] + mov eax, [r10] + mov [initial], eax + TEST 3 + TEST 9 + fninit + xor edi, edi + jmp exit +fail: + mov edi, 1 +exit: + mov eax, 60 + syscall + +section .rodata align=4 +cases: + ; Input, nearest-even, floor, ceil, truncation. + dd 0, 0, 0, 0, 0 + dd 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000 + dd 0x3f000000, 0, 0, 0x3f800000, 0 + dd 0xbf000000, 0x80000000, 0xbf800000, 0x80000000, 0x80000000 + dd 0x3fc00000, 0x40000000, 0x3f800000, 0x40000000, 0x3f800000 + dd 0xbfc00000, 0xc0000000, 0xc0000000, 0xbf800000, 0xbf800000 + dd 0x40200000, 0x40000000, 0x40000000, 0x40400000, 0x40000000 + dd 0xc0200000, 0xc0000000, 0xc0400000, 0xc0000000, 0xc0000000 + dd 0x4affffff, 0x4b000000, 0x4afffffe, 0x4b000000, 0x4afffffe + dd 0x4b800000, 0x4b800000, 0x4b800000, 0x4b800000, 0x4b800000 +control: dw 0x37e + +section .data align=32 +initial: + dd 0, 0x89abcdef + dq 0x123456789abcdef0, -1, -1 + +section .bss align=32 +output: resb 32 +mxcsr: resd 1 +mxcsr_before: resd 1 +status: resw 1 + +section .note.GNU-stack noalloc noexec nowrite progbits diff --git a/tests/integration/vshufpd-interleave.asm b/tests/integration/vshufpd-interleave.asm new file mode 100644 index 0000000000..2efffdac74 --- /dev/null +++ b/tests/integration/vshufpd-interleave.asm @@ -0,0 +1,88 @@ +; SPDX-FileCopyrightText: 2026 LAT Project Authors +; SPDX-License-Identifier: GPL-2.0-only +bits 64 +default rel +global _start + +section .text + +%macro CASE 3 + %assign narrow 0 + %ifidni %1, xmm + %assign narrow 1 + %endif + vmovdqu ymm0, [junk] + vmovdqu ymm1, [input_a] + vmovdqu ymm2, [input_b] + %if %3 == 7 + vmovdqu xmm1, [input_a] + vmovdqu xmm2, [input_b] + %endif + %if %3 == 0 || %3 == 7 + vshufpd %1 %+ 0, %1 %+ 1, %1 %+ 2, %2 + %elif %3 == 1 + vshufpd %1 %+ 1, %1 %+ 1, %1 %+ 2, %2 + vmovdqu ymm0, ymm1 + %elif %3 == 2 + vshufpd %1 %+ 2, %1 %+ 1, %1 %+ 2, %2 + vmovdqu ymm0, ymm2 + %elif %3 == 3 + vshufpd %1 %+ 0, %1 %+ 1, %1 %+ 1, %2 + %elif %3 == 4 + vshufpd %1 %+ 1, %1 %+ 1, %1 %+ 1, %2 + vmovdqu ymm0, ymm1 + %elif %3 == 5 + vshufpd %1 %+ 0, %1 %+ 1, [input_b], %2 + %else + vshufpd %1 %+ 1, %1 %+ 1, [input_b], %2 + vmovdqu ymm0, ymm1 + %endif + vmovdqu [actual], ymm0 + %assign half 0 + %rep 2 + %assign lane 0 + %rep 2 + %assign index ((%2 >> (half * 2 + lane)) & 1) + %if (narrow && half == 1) || (%3 == 7 && half == 1) + xor eax, eax + %elif lane == 0 || %3 == 3 || %3 == 4 + mov rax, [input_a + half * 16 + index * 8] + %else + mov rax, [input_b + half * 16 + index * 8] + %endif + cmp [actual + half * 16 + lane * 8], rax + jne fail + %assign lane lane + 1 + %endrep + %assign half half + 1 + %endrep +%endmacro + +_start: + mov edi, 1 + %assign immediate 0 + %rep 256 + %assign alias 0 + %rep 8 + CASE ymm, immediate, alias + CASE xmm, immediate, alias + %assign alias alias + 1 + %endrep + %assign immediate immediate + 1 + %endrep + xor edi, edi +fail: + mov eax, 60 + syscall + +section .rodata align=32 +input_a: dq 0x7ff0000000000001, 0x8000000000000000 + dq 0x0123456789abcdef, 0x1020304050607080 +input_b: dq 0x7ff8123456789abc, 0xffffffffffffffff + dq 0xfedcba9876543210, 0x8877665544332211 +junk: times 4 dq 0xdeadbeefdeadbeef + +section .bss align=32 +actual: resb 32 + +section .note.GNU-stack noalloc noexec nowrite progbits