Avx translation optimizations

accel/tcg/translate-all.c

@@ -424,7 +424,6 @@ int encode_search(TranslationBlock *tb, uint8_t *block)
 #ifdef CONFIG_LATX_OPT_PUSH_POP
 extern void ir2_opt_push_pop_fix(TranslationBlock *tb, CPUState *cpu, int i);
 #endif
-extern void opt_instptn_fix(CPUState *cpu, TranslationBlock *tb, int i);
 
 /* The cpu state corresponding to 'searched_pc' is restored.
  * When reset_icount is true, current TB will be interrupted and
@@ -486,9 +485,6 @@ static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
         ir2_opt_push_pop_fix(tb, cpu, (searched_pc - (uintptr_t)tb->tc.ptr) / 4);
     }
 #endif
-#ifdef CONFIG_LATX_INSTS_PATTERN
-    opt_instptn_fix(cpu, tb, i);
-#endif
 #ifdef CONFIG_PROFILER
     qatomic_set(&prof->restore_time,
                 prof->restore_time + profile_getclock() - ti);

target/i386/latx/include/insts-pattern.h

@@ -163,4 +163,9 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *ir1, int index,
 
 bool try_translate_instptn(IR1_INST *pir1);
 
+#ifdef CONFIG_LATX_INSTS_PATTERN
+unsigned instptn_flags_recovery(IR1_INST *pir1);
+void opt_instptn_fix(CPUArchState *env, unsigned recovery);
+#endif
+
 #endif

target/i386/latx/include/ir2.h

@@ -116,8 +116,5 @@ bool la_ir2_opcode_is_load(IR2_OPCODE opcode);
 void tr_ir2_optimize(TranslationBlock *tb);
 
 void ir2_opt_push_pop_fix(TranslationBlock *tb, CPUState *cpu, int i);
-#ifdef CONFIG_LATX_INSTS_PATTERN
-void opt_instptn_fix(CPUState *cpu, TranslationBlock *tb, int index);
-#endif
 void la_append_ir2_jmp_far(ADDR jmp_offset);
 #endif

target/i386/latx/include/latx-unwind.h

@@ -0,0 +1,29 @@
+/*
+ * SPDX-FileCopyrightText: 2021-2026 LAT Project Authors
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ */
+
+#ifndef LATX_UNWIND_H
+#define LATX_UNWIND_H
+
+/* Layout of the second per-instruction search-table word, also on i386. */
+#define LATX_UNWIND_CC_OP_MASK 0xff
+#define LATX_UNWIND_INSTPTN_SHIFT 8
+#define LATX_UNWIND_INSTPTN_MASK 0xff
+#define LATX_UNWIND_YMMH_SHIFT 16
+
+/* Delayed integer flags use the saved host a0/a1, never temporary IR1. */
+enum {
+    INSTPTN_FLAGS_NONE,
+    INSTPTN_FLAGS_CMP,
+    INSTPTN_FLAGS_TEST,
+    INSTPTN_FLAGS_BT,
+};
+
+#define INSTPTN_FLAGS_OP_MASK 3
+#define INSTPTN_FLAGS_SIZE_SHIFT 2
+#define INSTPTN_FLAGS_SIZE_MASK 3
+#define INSTPTN_FLAGS_SAME_REG (1 << 4)
+
+#endif

target/i386/latx/include/translate.h

@@ -1119,6 +1119,8 @@ TRANS_FUNC_DEF(vpshufb_lsx);
 TRANS_FUNC_DEF(vpsubx);
 TRANS_FUNC_DEF(vptest);
 TRANS_FUNC_DEF(vptest_lsx);
+void generate_avx_test_flags(IR1_INST *, IR2_OPND, IR2_OPND, IR2_OPND,
+                             uint8_t);
 TRANS_FUNC_DEF(vpunpckhxx);
 TRANS_FUNC_DEF(vpunpckhxx_lsx);
 TRANS_FUNC_DEF(vpunpcklxx);

target/i386/latx/optimization/flag-reduction.c

@@ -116,6 +116,9 @@ static const IR1_EFLAG_USEDEF ir1_opcode_eflag_usedef[] = {
     FLAG_DEFINE(VCOMISS, __NONE, __ALL_EFLAGS, __NONE),
     FLAG_DEFINE(VUCOMISD, __NONE, __ALL_EFLAGS, __NONE),
     FLAG_DEFINE(VUCOMISS, __NONE, __ALL_EFLAGS, __NONE),
+    FLAG_DEFINE(VPTEST,  __NONE, __ALL_EFLAGS, __NONE),
+    FLAG_DEFINE(VTESTPS, __NONE, __ALL_EFLAGS, __NONE),
+    FLAG_DEFINE(VTESTPD, __NONE, __ALL_EFLAGS, __NONE),
     FLAG_DEFINE(IRET,    __NONE, __INVALID, __NONE),
 
     /* Jcc */

target/i386/latx/optimization/insts-pattern.c

@@ -48,6 +48,9 @@ static inline bool ir1_can_pattern(IR1_INST *pir1)
     case WRAP(CDQ):
     case WRAP(TEST):
     case WRAP(UCOMISD):
+#ifdef CONFIG_LATX_AVX_OPT
+    case WRAP(VUCOMISD):
+#endif
 #ifdef CONFIG_LATX_SMC_OPT
     case WRAP(MOVAPS):
     case WRAP(MOVDQA):
@@ -101,6 +104,9 @@ static inline bool ir1_is_pattern_head(IR1_INST *pir1)
     case WRAP(CDQ):
     case WRAP(TEST):
     case WRAP(UCOMISD):
+#ifdef CONFIG_LATX_AVX_OPT
+    case WRAP(VUCOMISD):
+#endif
 #ifdef CONFIG_LATX_SMC_OPT
     case WRAP(MOVAPS):
     case WRAP(MOVDQA):
@@ -333,12 +339,23 @@ static int inst_pattern(TranslationBlock *tb,
             return 0;
         }
     case WRAP(UCOMISD):
+#ifdef CONFIG_LATX_AVX_OPT
+    case WRAP(VUCOMISD):
+#endif
         SCAN_CHECK(scan, 0);
         instptn_check_ucomisd_seta_0();
 
         ir1 = SCAN_IR1(tb, scan, 0);
+        if (ir1_addr_next(pir1) != ir1_addr(ir1)) {
+            return 0;
+        }
         switch (ir1_opcode(ir1)) {
         case WRAP(SETA):
+            /* A fused store has no separate SETA fault boundary. */
+            if (!ir1_opnd_is_gpr(ir1_get_opnd(ir1, 0))) {
+                ir1_set_eflag_def(pir1, 0x3f);
+                return 0;
+            }
             pir1->instptn.opc  = INSTPTN_OPC_UCOMISD_SETA;
             pir1->instptn.next = ir1;
             ir1->instptn.opc  = INSTPTN_OPC_NOP;
@@ -443,6 +460,32 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i
     IR1_OPND *opnd0 = NULL;
     IR1_OPND *opnd1 = NULL;
     switch (ir1_opcode(pir1)) {
+#ifdef CONFIG_LATX_AVX_OPT
+    case WRAP(VPTEST):
+    case WRAP(VTESTPS):
+    case WRAP(VTESTPD):
+        SCAN_CHECK(scan, 0);
+        ir1_jcc = SCAN_IR1(tb, scan, 0);
+        if (!option_enable_lasx || pir1_index + 1 != SCAN_IDX(scan, 0)) {
+            return false;
+        }
+        switch (ir1_opcode(ir1_jcc)) {
+        case WRAP(JE):
+        case WRAP(JNE):
+        case WRAP(JB):
+        case WRAP(JAE):
+        case WRAP(JBE):
+        case WRAP(JA):
+            instptn_check_test_jcc_0();
+            pir1->instptn.opc = INSTPTN_OPC_TEST_JCC;
+            pir1->instptn.next = ir1_jcc;
+            ir1_jcc->instptn.opc = INSTPTN_OPC_NOP;
+            break;
+        default:
+            break;
+        }
+        return false;
+#endif
         case WRAP(CMP):
         SCAN_CHECK(scan, 0);
         ir1_jcc = SCAN_IR1(tb, scan, 0);

target/i386/latx/translator/mem-interface.c

@@ -367,6 +367,24 @@ static IR2_OPND convert_mem_helper(IR1_OPND *opnd1, IR2_OPND *arg_dest_op,
     if (host_off) {
         if (si12_overflow(offset)) {
             *host_off = 0;
+#ifdef TARGET_X86_64
+            bool split_offset = CODEIS64 && ir1_addr_size(pir1) == 64 &&
+                                has_base && !has_index && !has_seg &&
+                                ir1_opnd_size(opnd1) != 80;
+#ifdef CONFIG_LATX_IMM_REG
+            split_offset = split_offset &&
+                           (!option_imm_reg || !option_imm_complex);
+#endif
+            if (split_offset) {
+                longx low = sextract64(offset, 0, 12);
+                longx high = offset - low;
+                if (low && high == (int32_t)high) {
+                    /* Keep the low displacement in the load/store itself. */
+                    *host_off = low;
+                    offset = high;
+                }
+            }
+#endif
         } else {
 #ifdef CONFIG_LATX_IMM_REG
             cache_skip = true;