fix(callgrind): model REP copies with calibrated bulk costs

CODSPEED-CHANGELOG.md

@@ -23,6 +23,13 @@ matching Linux on AArch64.
 
 ## Features
 
+### REP bulk cost model for `rep movs` / `rep stos`
+
+Callgrind now prices amd64 `rep movs` and `rep stos` from a calibrated
+hot-copy curve when cache simulation and `--cycle-estimation=yes` are both
+active: `Ct`/`Cl` carry the curve cost, and only data L1 misses are reported
+as `Dr`/`Dw`. `--rep-bulk-cost=no` restores the per-element model.
+
 ### Fixed Linux startup stack layout
 
 Reduce simulated cache-miss variation from argument and environment sizes

callgrind/Makefile.am

@@ -22,11 +22,14 @@ noinst_HEADERS = \
 	cycledecode_capstone.h \
 	sigkey.h \
 	events.h \
-	global.h
+	global.h \
+	rep_bulk.h
 
 # Generated cost tables (#included by cycledecode.c under CLG_WITH_CAPSTONE;
-# the arch is selected at compile time).
-EXTRA_DIST += x86_caps_lut.inc arm64_caps_lut.inc
+# the arch is selected at compile time) and the calibrated REP bulk curve
+# table included by rep_bulk.c (generated by the instruction cost calibration
+# pipeline).
+EXTRA_DIST += x86_caps_lut.inc arm64_caps_lut.inc rep_bulk_table.inc
 
 #----------------------------------------------------------------------------
 # callgrind-<platform>
@@ -52,6 +55,7 @@ CALLGRIND_SOURCES_COMMON = \
 	fn.c \
 	jumps.c \
 	main.c \
+	rep_bulk.c \
 	sim.c \
 	subprocess.c \
 	threads.c

callgrind/clo.c

@@ -568,6 +568,7 @@ Bool CLG_(process_cmd_line_option)(const HChar* arg)
    else if VG_BOOL_CLO(arg, "--branch-sim",      CLG_(clo).simulate_branch) {}
 
    else if VG_BOOL_CLO(arg, "--cycle-estimation", CLG_(clo).cycle_estimation) {}
+   else if VG_BOOL_CLO(arg, "--rep-bulk-cost",    CLG_(clo).rep_bulk_cost) {}
    else {
        Bool isCachesimOption = (*CLG_(cachesim).parse_opt)(arg);
 
@@ -617,6 +618,8 @@ void CLG_(print_usage)(void)
 "    --collect-jumps=no|yes    Collect jumps? [no]\n"
 "    --collect-bus=no|yes      Collect global bus events? [no]\n"
 "    --cycle-estimation=no|yes Estimate per-instruction cycles (Ct/Cl)? [no]\n"
+"    --rep-bulk-cost=no|yes    Calibrated rep movs/stos cost, needs cache\n"
+"                              sim and --cycle-estimation=yes [yes]\n"
 #if CLG_EXPERIMENTAL
 "    --collect-alloc=no|yes    Collect memory allocation info? [no]\n"
 #endif
@@ -704,6 +707,7 @@ void CLG_(set_clo_defaults)(void)
   CLG_(clo).simulate_cache = False;
   CLG_(clo).simulate_branch = False;
   CLG_(clo).cycle_estimation = False;
+  CLG_(clo).rep_bulk_cost = True;
 
   /* Call graph */
   CLG_(clo).pop_on_jump = False;

callgrind/docs/cl-manual.xml

@@ -995,6 +995,31 @@ Also see <xref linkend="cl-manual.cycles"/>.</para>
     </listitem>
   </varlistentry>
 
+  <varlistentry id="clopt.rep-bulk-cost" xreflabel="--rep-bulk-cost">
+    <term>
+      <option><![CDATA[--rep-bulk-cost=<yes|no> [default: yes] ]]></option>
+    </term>
+    <listitem>
+      <para>Model amd64 <computeroutput>rep movs</computeroutput> and
+      <computeroutput>rep stos</computeroutput> with a calibrated bulk cost
+      curve instead of the per-element model. Active only when cache
+      simulation and <option>--cycle-estimation=yes</option> are both
+      enabled, and not with <option>--simulate-wb=yes</option>,
+      <option>--cacheuse=yes</option> or non-amd64 guests.</para>
+      <para>For a modeled instruction, "Ct" and "Cl" both carry the curve
+      cost, and only data accesses that miss in LL are reported as "Dr"/"Dw"
+      (so Dr equals D1mr equals DLmr, likewise for writes): an L1 miss that
+      hits LL streams at near-hit cost in a REP copy. Instruction-side
+      counters and the cache simulation are unchanged.
+      <option>--rep-bulk-cost=no</option>
+      reproduces the per-element model exactly.</para>
+      <para>The curve was measured on disjoint, cache-resident buffers up to
+      16 KiB (MOVS) and 32 KiB (STOS); larger copies extend the last measured
+      slope. Overlapping or cache-miss-heavy copies are outside the
+      calibrated domain.</para>
+    </listitem>
+  </varlistentry>
+
   <varlistentry id="clopt.branch-sim" xreflabel="--branch-sim">
     <term>
       <option><![CDATA[--branch-sim=<yes|no> [default: no] ]]></option>

callgrind/global.h

@@ -128,6 +128,7 @@ struct _CommandLineOptions {
   Bool simulate_cache;      /* Call into cache simulator ? */
   Bool simulate_branch;     /* Call into branch prediction simulator ? */
   Bool cycle_estimation;    /* Estimate per-instruction cycles (Ct/Cl) ? */
+  Bool rep_bulk_cost;       /* Replace per-element REP cost by bulk curve ? */
 
   /* Call graph generation */
   Bool pop_on_jump;       /* Handle a jump between functions as ret+call */
@@ -279,6 +280,11 @@ struct _InstrInfo {
   UInt ct_incl;
   UInt cl_cost;
   UInt cl_incl;
+  /* REP bulk cost model classification (see rep_bulk.h). Only filled on
+   * first translation and only when the model is active; rep_bulk_op == 0
+   * means the instruction is not a supported rep-prefixed bulk copy. */
+  UChar rep_bulk_op;
+  UChar rep_elem_width;
 };
 
 
@@ -528,6 +534,14 @@ struct _exec_state {
   BBCC* nonskipped;
 
   Int call_stack_bottom; /* Index into fn_stack */
+
+  /* REP bulk cost model: the alignment class is latched once per REP
+   * operation from its first element, and the expected remaining count and
+   * destination detect the next operation. Saved/restored on thread and
+   * signal switches so an interrupted REP keeps its class. */
+  Addr       rep_next_dst; /* expected destination of the next element */
+  ULong      rep_next_rem; /* expected remaining count after last element */
+  UChar      rep_align;    /* CLG_REP_ALIGN_* latched at operation start */
 };
 
 /* Global state structures */
@@ -727,6 +741,10 @@ void CLG_(print_debug_usage)(void);
 
 /* from sim.c */
 void CLG_(init_eventsets)(void);
+/* REP bulk cost model: called at the IMark of each execution of a classified
+ * REP instruction. Args: inode, repeat count before VEX's decrement, source
+ * address (dst for STOS), destination address, guest DF. */
+void CLG_(log_rep_bulk)(InstrInfo*, ULong, Addr, Addr, Word);
 
 /* from main.c */
 Bool CLG_(get_debug_info)(Addr, const HChar **dirname,

callgrind/main.c

@@ -40,6 +40,15 @@
 #include "cg_branchpred.c"
 
 #include "cycledecode.h"
+#include "rep_bulk.h"
+
+#if defined(VGA_amd64)
+/* REP bulk cost model: VEX translates rep movs/stos on amd64 as one element
+ * per BB execution that jumps back to the instruction, and decrements RCX
+ * after its IMark. */
+#include "libvex_guest_amd64.h"
+#define CLG_GUEST_OFF(reg) offsetof(VexGuestAMD64State, guest_##reg)
+#endif
 
 /*------------------------------------------------------------*/
 /*--- Global variables                                     ---*/
@@ -397,6 +406,11 @@ static void flushEvents ( ClgState* clgs )
 	       // Ir event always is first for a guest instruction
 	       CLG_ASSERT(ev->inode->eventset == 0);
 	       ev->inode->eventset = CLG_(sets).base;
+	       /* REP bulk model: REP instructions carry per-instruction Ct/Cl
+	        * slots, filled at runtime from the calibrated bulk curve. */
+	       if (ev->inode->rep_bulk_op)
+		   ev->inode->eventset =
+		       CLG_(add_event_group)(ev->inode->eventset, EG_CYCLES);
 	       break;
 	   case Ev_Dr:
                // extend event set by Dr counters
@@ -597,6 +611,31 @@ static void flushEvents ( ClgState* clgs )
    clgs->events_used = 0;
 }
 
+/* REP bulk cost model: charge the element about to execute. At the IMark,
+ * before VEX's decrement, RCX counts the elements still to go. STOS has no
+ * source and passes RDI for both pointers. */
+static void addRepBulkCall ( ClgState* clgs, InstrInfo* inode )
+{
+#if defined(VGA_amd64)
+   Int     offs[4] = { CLG_GUEST_OFF(RCX), CLG_GUEST_OFF(RSI),
+                       CLG_GUEST_OFF(RDI), CLG_GUEST_OFF(DFLAG) };
+   IRExpr* args[4];
+   Int     i;
+
+   if (inode->rep_bulk_op == CLG_REP_OP_STOS)
+      offs[1] = CLG_GUEST_OFF(RDI);
+   for (i = 0; i < 4; i++) {
+      IRTemp t = newIRTemp(clgs->sbOut->tyenv, Ity_I64);
+      addStmtToIRSB(clgs->sbOut, IRStmt_WrTmp(t, IRExpr_Get(offs[i], Ity_I64)));
+      args[i] = IRExpr_RdTmp(t);
+   }
+   addStmtToIRSB(clgs->sbOut, IRStmt_Dirty(unsafeIRDirty_0_N(
+      0, "log_rep_bulk", VG_(fnptr_to_fnentry)(&CLG_(log_rep_bulk)),
+      mkIRExprVec_5(mkIRExpr_HWord((HWord)inode),
+                    args[0], args[1], args[2], args[3]))));
+#endif
+}
+
 static void addEvent_Ir ( ClgState* clgs, InstrInfo* inode )
 {
    Event* evt;
@@ -795,6 +834,8 @@ InstrInfo* next_InstrInfo ( ClgState* clgs, UInt instr_size )
        ii->ct_incl = 0;
        ii->cl_cost = 0;
        ii->cl_incl = 0;
+       ii->rep_bulk_op = 0;
+       ii->rep_elem_width = 0;
    }
 
    clgs->ii_index++;
@@ -1046,9 +1087,18 @@ IRSB* CLG_(instrument)( VgCallbackClosure* closure,
 	    // also use it.
 	    curr_inode = next_InstrInfo (&clgs, isize);
 
+        // Classify rep movs/stos for the REP bulk cost model. Their bulk
+        // curve replaces the per-instruction cycle estimate below.
+        if (CLG_(clo).rep_bulk_cost && !clgs.seen_before)
+            curr_inode->rep_bulk_op = clg_rep_bulk_classify(
+                (const UChar*)cia, isize, &curr_inode->rep_elem_width);
+        if (curr_inode->rep_bulk_op)
+            addRepBulkCall(&clgs, curr_inode);
+
         // Estimate this instruction's cycle cost by decoding the real
         // guest bytes. Only on first translation; values persist in the BB.
-        if (CLG_(clo).cycle_estimation && !clgs.seen_before) {
+        if (CLG_(clo).cycle_estimation && !clgs.seen_before
+            && !curr_inode->rep_bulk_op) {
             unsigned ct, cl;
             unsigned len = st->Ist.IMark.len ? st->Ist.IMark.len : isize;
             clg_cycle_cost((const unsigned char*)cia, len, &ct, &cl);

callgrind/rep_bulk.c

@@ -0,0 +1,118 @@
+/*--------------------------------------------------------------------*/
+/*--- REP bulk cost model: calibrated hot-copy curve.              ---*/
+/*---                                                    rep_bulk.c ---*/
+/*--------------------------------------------------------------------*/
+
+/* CodSpeed extension. See rep_bulk.h. */
+
+#include "rep_bulk.h"
+#include "pub_tool_libcassert.h"
+
+/* Generated by the calibration pipeline. Defines
+ * clg_rep_bulk_provenance[], clg_rep_bulk_pts_<id>[] and
+ * clg_rep_bulk_curves[] (types from rep_bulk.h). */
+#include "rep_bulk_table.inc"
+
+#define N_CURVES ((UInt)(sizeof(clg_rep_bulk_curves) / \
+                         sizeof(clg_rep_bulk_curves[0])))
+
+#define N_WIDTHS 4
+#define N_DIRS   2
+#define N_ALIGN  3
+
+/* The generated table lists the curves in key order: op, width, direction,
+ * alignment class. */
+static const ClgRepBulkCurve* curve(unsigned op, unsigned elem_width,
+                                    unsigned direction, unsigned align_class)
+{
+   UInt w = (UInt)__builtin_ctz(elem_width);
+   return &clg_rep_bulk_curves[(((op - CLG_REP_OP_MOVS) * N_WIDTHS + w)
+                                * N_DIRS + direction) * N_ALIGN + align_class];
+}
+
+/* Interpolate linearly between the neighbouring sampled byte sizes. Integer
+ * floor rounding keeps the result monotone, so marginal costs never become
+ * negative (consecutive element deltas telescope exactly for the same F). */
+static ULong cost_at(const ClgRepBulkCurve* c, ULong bytes)
+{
+   const ClgRepBulkPoint* p = c->pts;
+   UInt                   n = c->npts;
+
+   if (bytes <= p[0].bytes)
+      return p[0].centicycles;
+
+   /* Last point <= bytes. */
+   UInt lo = 0, hi = n - 1;
+   while (lo < hi) {
+      UInt mid = lo + (hi - lo + 1) / 2;
+      if (p[mid].bytes <= bytes)
+         lo = mid;
+      else
+         hi = mid - 1;
+   }
+
+   if (p[lo].bytes == bytes)
+      return p[lo].centicycles;
+
+   /* Beyond the last sampled size, extend the last segment's slope. */
+   if (lo + 1 == n)
+      lo--;
+
+   ULong dbytes = (ULong)(p[lo + 1].bytes - p[lo].bytes);
+   ULong dcost = p[lo + 1].centicycles - p[lo].centicycles;
+   ULong off = bytes - p[lo].bytes;
+   return p[lo].centicycles + (off * dcost) / dbytes;
+}
+
+void clg_rep_bulk_init(void)
+{
+   UInt i;
+
+   (void)clg_rep_bulk_provenance;
+   tl_assert(N_CURVES == 2 * N_WIDTHS * N_DIRS * N_ALIGN);
+   for (i = 0; i < N_CURVES; i++) {
+      const ClgRepBulkCurve* c = &clg_rep_bulk_curves[i];
+      tl_assert(c->npts >= 2);
+      tl_assert(curve(c->key.op, c->key.elem_width, c->key.direction,
+                      c->key.align_class) == c);
+   }
+}
+
+ULong clg_rep_bulk_marginal(unsigned op, unsigned elem_width,
+                            unsigned direction, unsigned align_class,
+                            ULong remaining_bytes, ULong elem_bytes)
+{
+   const ClgRepBulkCurve* c = curve(op, elem_width, direction, align_class);
+
+   /* F is monotone, so the difference is never negative and telescopes to
+    * F(total) over one operation. */
+   return cost_at(c, remaining_bytes) - cost_at(c, remaining_bytes - elem_bytes);
+}
+
+/* Prefixes, then A4/A5 (MOVS) or AA/AB (STOS) as the last byte; REX must
+ * directly precede the opcode. The 67 prefix (ECX/ESI/EDI) is not modeled:
+ * the helper reads the full 64-bit registers. */
+ClgRepBulkOp clg_rep_bulk_classify(const UChar* bytes, UInt len,
+                                   UChar* elem_width)
+{
+   UChar opc = bytes[len - 1];
+   Bool  rep = False, opsz16 = False, rex_w = False;
+   UInt  i;
+
+   for (i = 0; i + 1 < len; i++) {
+      switch (bytes[i]) {
+      case 0xF3: rep = True; break;
+      case 0x66: opsz16 = True; break;
+      default:
+         if ((bytes[i] & 0xF0) != 0x40 || i + 2 != len)
+            return CLG_REP_OP_NONE;
+         rex_w = (bytes[i] & 0x08) != 0;
+      }
+   }
+
+   if (!rep || ((opc & ~1) != 0xA4 && (opc & ~1) != 0xAA))
+      return CLG_REP_OP_NONE;
+   /* Even opcodes copy bytes; odd ones use the operand size. */
+   *elem_width = (opc & 1) == 0 ? 1 : rex_w ? 8 : opsz16 ? 2 : 4;
+   return (opc & ~1) == 0xA4 ? CLG_REP_OP_MOVS : CLG_REP_OP_STOS;
+}