From 09053ba2cbd98162a6ae51e535b222ed3c0cacf4 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 5 Nov 2026 11:30:09 +0100 Subject: [PATCH 1/8] fix(callgrind): model REP copies with calibrated bulk costs Replace per-element REP MOVS/STOS cycle estimates with measured hot-copy curves on amd64. Charge the calibrated marginal through the existing Ct/Cl counters and report only data L1 misses as Dr/Dw, retaining cache simulation, instruction counters, and all miss penalties. Enable the model with cache simulation and cycle estimation; --rep-bulk-cost=no restores the per-element accounting. Keep the output format unchanged and document mixed-counter, overlap, cache-residency, and vector-comparison limitations. The curves come from a Zen 5 calibration generated out of tree. Each curve includes the fixed REP startup cost measured against a no-REP baseline. Points that do not change the interpolated cost at any element multiple are dropped, so the table stores 2,567 points across 48 curves. Add default and legacy runtime regressions for zero-length copies, hot/cold counters, backward overlap, stores, and self-copy. --- CODSPEED-CHANGELOG.md | 7 + callgrind/Makefile.am | 10 +- callgrind/clo.c | 4 + callgrind/docs/cl-manual.xml | 23 + callgrind/global.h | 22 + callgrind/main.c | 87 ++ callgrind/rep_bulk.c | 209 +++++ callgrind/rep_bulk.h | 82 ++ callgrind/rep_bulk_table.inc | 880 +++++++++++++++++++++ callgrind/sim.c | 114 ++- callgrind/tests/Makefile.am | 11 +- callgrind/tests/rep_bulk.c | 145 ++++ callgrind/tests/rep_bulk.post.exp | 10 + callgrind/tests/rep_bulk.stderr.exp | 0 callgrind/tests/rep_bulk.vgtest | 5 + callgrind/tests/rep_bulk_check.pl | 122 +++ callgrind/tests/rep_bulk_legacy.post.exp | 8 + callgrind/tests/rep_bulk_legacy.stderr.exp | 0 callgrind/tests/rep_bulk_legacy.vgtest | 5 + callgrind/threads.c | 9 + 20 files changed, 1740 insertions(+), 13 deletions(-) create mode 100644 callgrind/rep_bulk.c create mode 100644 callgrind/rep_bulk.h create mode 100644 callgrind/rep_bulk_table.inc create mode 100644 callgrind/tests/rep_bulk.c create mode 100644 callgrind/tests/rep_bulk.post.exp create mode 100644 callgrind/tests/rep_bulk.stderr.exp create mode 100644 callgrind/tests/rep_bulk.vgtest create mode 100755 callgrind/tests/rep_bulk_check.pl create mode 100644 callgrind/tests/rep_bulk_legacy.post.exp create mode 100644 callgrind/tests/rep_bulk_legacy.stderr.exp create mode 100644 callgrind/tests/rep_bulk_legacy.vgtest diff --git a/CODSPEED-CHANGELOG.md b/CODSPEED-CHANGELOG.md index 61808930c4..63f3fdde9d 100644 --- a/CODSPEED-CHANGELOG.md +++ b/CODSPEED-CHANGELOG.md @@ -23,6 +23,13 @@ matching Linux on AArch64. ## Features +### REP bulk cost model for `rep movs` / `rep stos` + +Callgrind now prices amd64 `rep movs` and `rep stos` from a calibrated +hot-copy curve when cache simulation and `--cycle-estimation=yes` are both +active: `Ct`/`Cl` carry the curve cost, and only data L1 misses are reported +as `Dr`/`Dw`. `--rep-bulk-cost=no` restores the per-element model. + ### Fixed Linux startup stack layout Reduce simulated cache-miss variation from argument and environment sizes diff --git a/callgrind/Makefile.am b/callgrind/Makefile.am index de92e16b89..0b90003128 100644 --- a/callgrind/Makefile.am +++ b/callgrind/Makefile.am @@ -22,11 +22,14 @@ noinst_HEADERS = \ cycledecode_capstone.h \ sigkey.h \ events.h \ - global.h + global.h \ + rep_bulk.h # Generated cost tables (#included by cycledecode.c under CLG_WITH_CAPSTONE; -# the arch is selected at compile time). -EXTRA_DIST += x86_caps_lut.inc arm64_caps_lut.inc +# the arch is selected at compile time) and the calibrated REP bulk curve +# table included by rep_bulk.c (generated by the instruction cost calibration +# pipeline). +EXTRA_DIST += x86_caps_lut.inc arm64_caps_lut.inc rep_bulk_table.inc #---------------------------------------------------------------------------- # callgrind- @@ -52,6 +55,7 @@ CALLGRIND_SOURCES_COMMON = \ fn.c \ jumps.c \ main.c \ + rep_bulk.c \ sim.c \ subprocess.c \ threads.c diff --git a/callgrind/clo.c b/callgrind/clo.c index 911051a27d..8549049a2c 100644 --- a/callgrind/clo.c +++ b/callgrind/clo.c @@ -568,6 +568,7 @@ Bool CLG_(process_cmd_line_option)(const HChar* arg) else if VG_BOOL_CLO(arg, "--branch-sim", CLG_(clo).simulate_branch) {} else if VG_BOOL_CLO(arg, "--cycle-estimation", CLG_(clo).cycle_estimation) {} + else if VG_BOOL_CLO(arg, "--rep-bulk-cost", CLG_(clo).rep_bulk_cost) {} else { Bool isCachesimOption = (*CLG_(cachesim).parse_opt)(arg); @@ -617,6 +618,8 @@ void CLG_(print_usage)(void) " --collect-jumps=no|yes Collect jumps? [no]\n" " --collect-bus=no|yes Collect global bus events? [no]\n" " --cycle-estimation=no|yes Estimate per-instruction cycles (Ct/Cl)? [no]\n" +" --rep-bulk-cost=no|yes Calibrated rep movs/stos cost, needs cache\n" +" sim and --cycle-estimation=yes [yes]\n" #if CLG_EXPERIMENTAL " --collect-alloc=no|yes Collect memory allocation info? [no]\n" #endif @@ -704,6 +707,7 @@ void CLG_(set_clo_defaults)(void) CLG_(clo).simulate_cache = False; CLG_(clo).simulate_branch = False; CLG_(clo).cycle_estimation = False; + CLG_(clo).rep_bulk_cost = True; /* Call graph */ CLG_(clo).pop_on_jump = False; diff --git a/callgrind/docs/cl-manual.xml b/callgrind/docs/cl-manual.xml index 8b4e50d853..148ee883d6 100644 --- a/callgrind/docs/cl-manual.xml +++ b/callgrind/docs/cl-manual.xml @@ -995,6 +995,29 @@ Also see . + + + + + + Model amd64 rep movs and + rep stos with a calibrated bulk cost + curve instead of the per-element model. Active only when cache + simulation and are both + enabled, and not with , + or non-amd64 guests. + For a modeled instruction, "Ct" and "Cl" both carry the curve + cost, and only data accesses that miss in L1 are reported as "Dr"/"Dw" + (so Dr equals D1mr and Dw equals D1mw); instruction-side counters and + the cache simulation are unchanged. + reproduces the per-element model exactly. + The curve was measured on disjoint, cache-resident buffers up to + 16 KiB (MOVS) and 32 KiB (STOS); larger copies extend the last measured + slope. Overlapping or cache-miss-heavy copies are outside the + calibrated domain. + + + diff --git a/callgrind/global.h b/callgrind/global.h index 97a22bf252..1727e50c0b 100644 --- a/callgrind/global.h +++ b/callgrind/global.h @@ -128,6 +128,7 @@ struct _CommandLineOptions { Bool simulate_cache; /* Call into cache simulator ? */ Bool simulate_branch; /* Call into branch prediction simulator ? */ Bool cycle_estimation; /* Estimate per-instruction cycles (Ct/Cl) ? */ + Bool rep_bulk_cost; /* Replace per-element REP cost by bulk curve ? */ /* Call graph generation */ Bool pop_on_jump; /* Handle a jump between functions as ret+call */ @@ -279,6 +280,11 @@ struct _InstrInfo { UInt ct_incl; UInt cl_cost; UInt cl_incl; + /* REP bulk cost model classification (see rep_bulk.h). Only filled on + * first translation and only when the model is active; rep_bulk_op == 0 + * means the instruction is not a supported rep-prefixed bulk copy. */ + UChar rep_bulk_op; + UChar rep_elem_width; }; @@ -528,6 +534,14 @@ struct _exec_state { BBCC* nonskipped; Int call_stack_bottom; /* Index into fn_stack */ + + /* REP bulk cost model: the alignment class is latched once per REP + * operation from its first element, and the expected remaining count and + * destination detect the next operation. Saved/restored on thread and + * signal switches so an interrupted REP keeps its class. */ + Addr rep_next_dst; /* expected destination of the next element */ + ULong rep_next_rem; /* expected remaining count after last element */ + UChar rep_align; /* CLG_REP_ALIGN_* latched at operation start */ }; /* Global state structures */ @@ -687,10 +701,18 @@ struct cachesim_if void (*log_0I1Dr)(InstrInfo*, Addr, Word) VG_REGPARM(3); void (*log_0I1Dw)(InstrInfo*, Addr, Word) VG_REGPARM(3); + /* REP bulk cost model: called once per executed element of a supported + * rep-prefixed bulk copy. Args: inode, repeat count after the decrement + * of this element, source address (dst if none), destination address, + * guest DF flag. Zero when the model is off. Uses the ordinary amd64 + * calling convention (regparms=0), not a register-parm ABI. */ + void (*log_rep_bulk)(InstrInfo*, ULong, Addr, Addr, Word); + // function names of helpers (for debugging generated code) const HChar *log_1I0D_name, *log_2I0D_name, *log_3I0D_name; const HChar *log_1I1Dr_name, *log_1I1Dw_name; const HChar *log_0I1Dr_name, *log_0I1Dw_name; + const HChar *log_rep_bulk_name; }; // Event groups diff --git a/callgrind/main.c b/callgrind/main.c index 69af08255f..a9b2b572e3 100644 --- a/callgrind/main.c +++ b/callgrind/main.c @@ -40,6 +40,17 @@ #include "cg_branchpred.c" #include "cycledecode.h" +#include "rep_bulk.h" + +#if defined(VGA_amd64) +/* REP bulk cost model: read the repeat counter and DF from the guest state. + * VEX translates rep movs/stos on amd64 as one element per BB execution with + * an unconditional jump back to the instruction, and uses RCX (or ECX with a + * 67 prefix) as the repeat counter, decremented before the element access. */ +#include "libvex_guest_amd64.h" +#define CLG_REP_COUNT_OFF offsetof(VexGuestAMD64State, guest_RCX) +#define CLG_REP_DFLAG_OFF offsetof(VexGuestAMD64State, guest_DFLAG) +#endif /*------------------------------------------------------------*/ /*--- Global variables ---*/ @@ -397,6 +408,11 @@ static void flushEvents ( ClgState* clgs ) // Ir event always is first for a guest instruction CLG_ASSERT(ev->inode->eventset == 0); ev->inode->eventset = CLG_(sets).base; + /* REP bulk model: REP instructions carry per-instruction Ct/Cl + * slots, filled at runtime from the calibrated bulk curve. */ + if (ev->inode->rep_bulk_op) + ev->inode->eventset = + CLG_(add_event_group)(ev->inode->eventset, EG_CYCLES); break; case Ev_Dr: // extend event set by Dr counters @@ -594,6 +610,59 @@ static void flushEvents ( ClgState* clgs ) addStmtToIRSB( clgs->sbOut, IRStmt_Dirty(di) ); } + /* REP bulk cost model: charge one calibrated bulk delta per executed + * element. Emitted after the normal memory helpers; the cache simulation + * is unchanged. `Get(RCX)` is the repeat count + * *after* VEX's decrement for this element (VEX decrements before the + * access), so the helper adds 1 back. */ +#if defined(VGA_amd64) + if (CLG_(cachesim).log_rep_bulk) { + InstrInfo* rii = 0; + IRExpr* src = 0; + IRExpr* dst = 0; + Int j; + + for (j = clgs->events_used - 1; j >= 0; j--) { + Event* e = &clgs->events[j]; + if (e->tag != Ev_Dr && e->tag != Ev_Dw && e->tag != Ev_Dm) + continue; + if (!rii) + rii = e->inode; + if (e->inode != rii) + continue; + if (e->tag == Ev_Dr) { + if (!src) + src = e->Ev.Dr.ea; + } else { + if (!dst) + dst = get_Event_dea(e); + } + } + + if (rii && rii->rep_bulk_op && dst) { + /* The two addresses only classify alignment once per operation; + * STOS and merged read-modify events have no separate source. + * A 67-prefixed REP needs no mask: VEX writes the decremented ECX + * zero-extended into RCX before the element's access. */ + IRTemp count_tmp = newIRTemp(clgs->sbOut->tyenv, Ity_I64); + IRTemp dflag_tmp = newIRTemp(clgs->sbOut->tyenv, Ity_I64); + addStmtToIRSB(clgs->sbOut, IRStmt_WrTmp( + count_tmp, IRExpr_Get(CLG_REP_COUNT_OFF, Ity_I64))); + addStmtToIRSB(clgs->sbOut, IRStmt_WrTmp( + dflag_tmp, IRExpr_Get(CLG_REP_DFLAG_OFF, Ity_I64))); + + di = unsafeIRDirty_0_N( + 0, CLG_(cachesim).log_rep_bulk_name, + VG_(fnptr_to_fnentry)(CLG_(cachesim).log_rep_bulk), + mkIRExprVec_5( mkIRExpr_HWord((HWord)rii), + IRExpr_RdTmp(count_tmp), + src ? src : dst, dst, + IRExpr_RdTmp(dflag_tmp) ) ); + addStmtToIRSB( clgs->sbOut, IRStmt_Dirty(di) ); + } + } +#endif + clgs->events_used = 0; } @@ -795,6 +864,8 @@ InstrInfo* next_InstrInfo ( ClgState* clgs, UInt instr_size ) ii->ct_incl = 0; ii->cl_cost = 0; ii->cl_incl = 0; + ii->rep_bulk_op = 0; + ii->rep_elem_width = 0; } clgs->ii_index++; @@ -1055,6 +1126,22 @@ IRSB* CLG_(instrument)( VgCallbackClosure* closure, curr_inode->ct_cost = ct; curr_inode->cl_cost = cl; } + + // Classify supported rep-prefixed bulk copies (rep movs/rep stos) for + // the REP bulk cost model. The decision is static per instruction. + if (CLG_(clo).rep_bulk_cost && !clgs.seen_before) { + ClgRepBulkInfo ri; + clg_rep_bulk_classify((const unsigned char*)cia, isize, &ri); + curr_inode->rep_bulk_op = ri.op; + curr_inode->rep_elem_width = ri.elem_width; + if (ri.op != CLG_REP_OP_NONE) { + /* The calibrated bulk curve replaces this instruction's + * static per-element cycle estimate completely: drop it from + * both the dump-time charge and the inclusive prefix sums. */ + curr_inode->ct_cost = 0; + curr_inode->cl_cost = 0; + } + } addEvent_Ir( &clgs, curr_inode ); break; } diff --git a/callgrind/rep_bulk.c b/callgrind/rep_bulk.c new file mode 100644 index 0000000000..0688381d83 --- /dev/null +++ b/callgrind/rep_bulk.c @@ -0,0 +1,209 @@ +/*--------------------------------------------------------------------*/ +/*--- REP bulk cost model: calibrated hot-copy curve. ---*/ +/*--- rep_bulk.c ---*/ +/*--------------------------------------------------------------------*/ + +/* CodSpeed extension. See rep_bulk.h. */ + +#include "rep_bulk.h" + +/* Generated by the calibration pipeline. Defines + * clg_rep_bulk_provenance[], clg_rep_bulk_pts_[] and + * clg_rep_bulk_curves[] (types from rep_bulk.h). */ +#include "rep_bulk_table.inc" + +#define N_CURVES ((UInt)(sizeof(clg_rep_bulk_curves) / \ + sizeof(clg_rep_bulk_curves[0]))) + +#define N_WIDTHS 4 +#define N_DIRS 2 +#define N_ALIGN 3 + +/* Exact-key index, resolved once at startup. The model is enabled only if + * every key is present. */ +static const ClgRepBulkCurve* curve_idx[CLG_REP_OP_STOS + 1][N_WIDTHS] + [N_DIRS][N_ALIGN]; + +static Int width_index(unsigned elem_width) +{ + switch (elem_width) { + case 1: return 0; + case 2: return 1; + case 4: return 2; + case 8: return 3; + default: return -1; + } +} + +static Bool key_valid(const ClgRepBulkKey* k) +{ + return (k->op == CLG_REP_OP_MOVS || k->op == CLG_REP_OP_STOS) + && width_index(k->elem_width) >= 0 + && k->direction < N_DIRS + && k->align_class < N_ALIGN; +} + +/* Interpolate linearly between the neighbouring sampled byte sizes. Integer + * floor rounding keeps the result monotone, so marginal costs never become + * negative (consecutive element deltas telescope exactly for the same F). */ +static ULong cost_at(const ClgRepBulkCurve* c, ULong bytes) +{ + const ClgRepBulkPoint* p = c->pts; + UInt n = c->npts; + + if (bytes <= p[0].bytes) + return p[0].centicycles; + + /* Last point <= bytes. */ + UInt lo = 0, hi = n - 1; + while (lo < hi) { + UInt mid = lo + (hi - lo + 1) / 2; + if (p[mid].bytes <= bytes) + lo = mid; + else + hi = mid - 1; + } + + if (p[lo].bytes == bytes) + return p[lo].centicycles; + + /* Beyond the last sampled size, extend the last segment's slope. */ + if (lo + 1 == n) + lo--; + + ULong dbytes = (ULong)(p[lo + 1].bytes - p[lo].bytes); + ULong dcost = p[lo + 1].centicycles - p[lo].centicycles; + ULong off = bytes - p[lo].bytes; + return p[lo].centicycles + (off * dcost) / dbytes; +} + +UInt clg_rep_bulk_init(void) +{ + UInt bad = 0, i, w, d, a; + + /* Table validation: keys, F(0)==0, ascending sizes, monotone costs. */ + for (i = 0; i < N_CURVES; i++) { + const ClgRepBulkCurve* c = &clg_rep_bulk_curves[i]; + UInt j; + + if (!key_valid(&c->key) || c->npts < 2) { + bad++; + continue; + } + if (c->pts[0].bytes != 0 || c->pts[0].centicycles != 0) { + bad++; + continue; + } + for (j = 1; j < c->npts; j++) { + if (c->pts[j].bytes <= c->pts[j - 1].bytes + || c->pts[j].centicycles < c->pts[j - 1].centicycles) { + bad++; + break; + } + } + if (j != c->npts) + continue; + + curve_idx[c->key.op][width_index(c->key.elem_width)] + [c->key.direction][c->key.align_class] = c; + } + + /* Every key is required: MOVS and STOS, all four widths, forward and + * backward, all three alignment classes. There is no cross-direction or + * cross-class substitution; the caller disables the model if any key is + * missing so the legacy per-element model stays in effect. */ + for (w = 0; w < N_WIDTHS; w++) + for (d = 0; d < N_DIRS; d++) + for (a = 0; a < N_ALIGN; a++) { + if (!curve_idx[CLG_REP_OP_MOVS][w][d][a]) + bad++; + if (!curve_idx[CLG_REP_OP_STOS][w][d][a]) + bad++; + } + + /* Keep the provenance string referenced (it documents the artifact). */ + if (clg_rep_bulk_provenance[0] == 0) + bad++; + + return bad; +} + +ULong clg_rep_bulk_marginal(unsigned op, unsigned elem_width, + unsigned direction, unsigned align_class, + ULong remaining_bytes, ULong elem_bytes) +{ + const ClgRepBulkCurve* c = + curve_idx[op][width_index(elem_width)][direction][align_class]; + + /* F is monotone, so the difference is never negative and telescopes to + * F(total) over one operation. */ + return cost_at(c, remaining_bytes) - cost_at(c, remaining_bytes - elem_bytes); +} + +/* Prefixes followed by A4/A5/AA/AB; REX must directly precede the opcode. */ +void clg_rep_bulk_classify(const unsigned char* bytes, unsigned len, + ClgRepBulkInfo* out) +{ + out->op = CLG_REP_OP_NONE; + out->elem_width = 0; +#if defined(VGA_amd64) + unsigned i = 0; + int rep = 0, opsz16 = 0, addr32p = 0, rex_w = 0; + + if (!bytes || len < 2 || len > 15) + return; + + while (i < len) { + unsigned char c = bytes[i]; + if (c == 0xF3 && !rep) { + rep = 1; + i++; + continue; + } + if (c == 0xF2) + return; /* REPNE: never a supported bulk copy */ + if (c == 0x66 && !opsz16) { + opsz16 = 1; + i++; + continue; + } + if (c == 0x67 && !addr32p) { + addr32p = 1; + i++; + continue; + } + if (c >= 0x40 && c <= 0x4F) { + rex_w = (c & 0x08) != 0; + i++; + } + break; + } + + /* Exactly the opcode must follow the prefix run: no operands, no + * trailing bytes (the IMark length always covers the whole insn). */ + if (!rep || i + 1 != len) + return; + + switch (bytes[i]) { + case 0xA4: /* movsb */ + out->op = CLG_REP_OP_MOVS; + out->elem_width = 1; + break; + case 0xA5: /* movsw/movsd/movsq */ + out->op = CLG_REP_OP_MOVS; + out->elem_width = rex_w ? 8 : (opsz16 ? 2 : 4); + break; + case 0xAA: /* stosb */ + out->op = CLG_REP_OP_STOS; + out->elem_width = 1; + break; + case 0xAB: /* stosw/stosd/stosq */ + out->op = CLG_REP_OP_STOS; + out->elem_width = rex_w ? 8 : (opsz16 ? 2 : 4); + break; + } +#else + (void)bytes; + (void)len; +#endif +} diff --git a/callgrind/rep_bulk.h b/callgrind/rep_bulk.h new file mode 100644 index 0000000000..b09b1a8e91 --- /dev/null +++ b/callgrind/rep_bulk.h @@ -0,0 +1,82 @@ +/*--------------------------------------------------------------------*/ +/*--- REP bulk cost model: calibrated hot-copy curve. ---*/ +/*--- rep_bulk.h ---*/ +/*--------------------------------------------------------------------*/ + +/* CodSpeed extension. + * + * Replaces the per-element cost of rep-prefixed MOVS/STOS with a calibrated + * curve F: the hot-cache cost of one instruction over a given number of + * bytes, in centi-cycles. Each executed element adds + * + * F(remaining_bytes) - F(remaining_bytes - elem_bytes) + * + * which telescopes to F(total_bytes) and stays exact across signal + * interruption, because "remaining" is read from the guest repeat counter. + */ + +#ifndef CLG_REP_BULK_H +#define CLG_REP_BULK_H + +#include "pub_tool_basics.h" + +typedef enum { + CLG_REP_OP_NONE = 0, + CLG_REP_OP_MOVS = 1, /* rep movs{b,w,d,q}: one read + one write per element */ + CLG_REP_OP_STOS = 2 /* rep stos{b,w,d,q}: one write per element */ +} ClgRepBulkOp; + +typedef struct { + unsigned char op; /* ClgRepBulkOp */ + unsigned char elem_width; /* element size in bytes (1,2,4,8), 0 if none */ +} ClgRepBulkInfo; + +/* From the guest DF flag, which is constant during an instruction. */ +#define CLG_REP_DIR_FORWARD 0 +#define CLG_REP_DIR_BACKWARD 1 + +/* Alignment class, latched from the first element's pointers (STOS uses + * dst as src): 0 both %64==0, 1 both %64==8, 2 other (conservative). */ +#define CLG_REP_ALIGN_ALIGNED 0 +#define CLG_REP_ALIGN_SAME_UNALIGN 1 +#define CLG_REP_ALIGN_OTHER 2 + +typedef struct { + unsigned char op; /* ClgRepBulkOp */ + unsigned char elem_width; /* 1, 2, 4 or 8 bytes */ + unsigned char direction; /* CLG_REP_DIR_* */ + unsigned char align_class; /* CLG_REP_ALIGN_* */ +} ClgRepBulkKey; + +typedef struct { + UInt bytes; /* cumulative element bytes, strictly ascending */ + ULong centicycles; /* total hot-resident cost, centi-cycles */ +} ClgRepBulkPoint; + +typedef struct { + ClgRepBulkKey key; + const ClgRepBulkPoint* pts; + UInt npts; +} ClgRepBulkCurve; + +/* Build the lookup index and check the table. Returns the number of table + * problems (0 == clean); the caller reports them. Call once at startup; + * clg_rep_bulk_marginal is only valid after it returned 0. */ +UInt clg_rep_bulk_init(void); + +/* Marginal bulk cost of the element at `remaining_bytes` (bytes still to + * execute before this element, a nonzero multiple of elem_bytes), in + * centi-cycles. The key must be a classified (op, width) pair with a + * CLG_REP_DIR_* and CLG_REP_ALIGN_* value; beyond the last calibrated + * point the last segment's slope is extended. */ +ULong clg_rep_bulk_marginal(unsigned op, unsigned elem_width, + unsigned direction, unsigned align_class, + ULong remaining_bytes, ULong elem_bytes); + +/* Classify the amd64 guest instruction in [bytes, bytes+len) by its + * encoding (no Capstone needed). Always initialises *out; out->op is + * CLG_REP_OP_NONE unless it is a rep-prefixed MOVS or STOS. */ +void clg_rep_bulk_classify(const unsigned char* bytes, unsigned len, + ClgRepBulkInfo* out); + +#endif /* CLG_REP_BULK_H */ diff --git a/callgrind/rep_bulk_table.inc b/callgrind/rep_bulk_table.inc new file mode 100644 index 0000000000..fa1bff0b2f --- /dev/null +++ b/callgrind/rep_bulk_table.inc @@ -0,0 +1,880 @@ +/* Generated by lut-gen/x86/repl_bulk/repl_bulk.py -- do not edit. + * + * Schema (from callgrind/rep_bulk.h): + * ClgRepBulkPoint { unsigned bytes; unsigned long long centicycles; } + * ClgRepBulkCurve { ClgRepBulkKey key; const ClgRepBulkPoint *pts; + * unsigned npts; } + * + * F(bytes) is the cumulative hot L1-resident total cost of ONE rep-prefixed + * instruction, in centi-cycles (1 cycle = 100), from dispatch through + * completion. It excludes surrounding code and every L1/LL/RAM miss penalty. + * Per executed element (width w): F(rem*w) - F((rem-1)*w), telescoping. + * Between points the consumer uses integer-linear floor interpolation. + * Beyond the last point cost_at extends the last segment linearly. + * align 0 = both 64B aligned, 1 = both offset 8, 2 = other (conservative + * envelope of odd/split measurements; see provenance). + * + * raw sha256: 17c25bd00601968d162a82a50994c1f293947cb7872cf35566f113a36b362737 + * probe sha256: 2f824fc61dd68a6d229a84091c99f4e445ea832a82f7454e3a80e6211a8903c8 + * Calibration: AMD Ryzen 9 9950X 16-Core Processor, measured fenced completion cost. + * This is NOT the static Zen4 uops.info LUT. + * calibrated max: MOVS 16384 bytes, STOS 32768 bytes + */ +static const char clg_rep_bulk_provenance[] = "repl-bulk-v1 host=AMD Ryzen 9 9950X 16-Core Processor date=2026-11-05T10:44:52.369756+00:00 raw_sha256=17c25bd00601968d162a82a50994c1f293947cb7872cf35566f113a36b362737 probe_sha256=2f824fc61dd68a6d229a84091c99f4e445ea832a82f7454e3a80e6211a8903c8 toolchain=gcc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0 calibrated_max=MOVS:16384,STOS:32768 units=centi-cycles(1cycle=100) regime=hot-L1-resident align2=conservative-envelope >max=carry-linear-tail domain=disjoint-buffers(overlap-not-claimed) cache-miss-penalties-excluded"; + +static const ClgRepBulkPoint clg_rep_bulk_pts_0[] = { + {0,0}, {1,1125}, {127,1125}, {128,3714}, + {192,3714}, {224,3746}, {320,3746}, {384,3820}, + {448,3900}, {640,3900}, {768,4068}, {896,4215}, + {1024,4521}, {1280,4876}, {1536,5228}, {1792,5666}, + {2048,6078}, {2560,6980}, {3072,7784}, {3584,8545}, + {4096,9319}, {5120,10995}, {6144,12645}, {7168,14223}, + {8192,15831}, {10240,19052}, {12288,22270}, {14336,25445}, + {16384,29317}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_1[] = { + {0,0}, {1,1096}, {64,1096}, {65,1099}, + {127,1099}, {128,4198}, {160,4198}, {192,4286}, + {224,4286}, {256,4496}, {320,4527}, {384,4739}, + {448,4913}, {512,5062}, {640,5264}, {768,5458}, + {896,5659}, {1024,5813}, {1280,6261}, {1536,6657}, + {1792,7016}, {2048,7396}, {2560,8042}, {3072,9015}, + {3584,9715}, {4096,10565}, {5120,12180}, {6144,13724}, + {7168,15376}, {8192,17047}, {10240,20188}, {12288,23366}, + {14336,26560}, {16384,30398}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_2[] = { + {0,0}, {1,4916}, {2,4916}, {3,5072}, + {4,5072}, {5,5165}, {8,5165}, {9,5205}, + {10,5205}, {11,5273}, {16,5273}, {17,5320}, + {18,5320}, {19,5346}, {20,5346}, {21,5348}, + {25,5349}, {48,5349}, {56,5589}, {63,5728}, + {64,5994}, {65,6026}, {127,6026}, {128,7875}, + {129,7883}, {160,7900}, {192,8018}, {224,8018}, + {256,8135}, {320,8209}, {384,8333}, {448,8415}, + {512,8572}, {640,8720}, {768,8944}, {896,9093}, + {1024,9379}, {1280,9795}, {1536,10348}, {1792,10999}, + {2048,12274}, {2560,13642}, {3072,14981}, {3584,16649}, + {4096,20263}, {5120,23455}, {6144,25200}, {7168,27963}, + {8192,31800}, {10240,37889}, {12288,43611}, {14336,48748}, + {16384,55631}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_3[] = { + {0,0}, {1,10338}, {2,10448}, {3,10544}, + {4,10647}, {5,10735}, {6,10844}, {8,11034}, + {9,11133}, {10,11248}, {12,11440}, {14,11639}, + {15,11939}, {16,12042}, {17,12141}, {20,12437}, + {23,12741}, {24,12835}, {25,12933}, {27,13138}, + {28,13250}, {30,13434}, {31,13540}, {32,13640}, + {33,13739}, {40,14431}, {48,15236}, {56,16036}, + {63,16733}, {64,16831}, {65,16932}, {80,18435}, + {96,20026}, {112,21622}, {127,23125}, {128,23218}, + {129,23318}, {160,26414}, {192,29630}, {224,32818}, + {256,36004}, {320,43111}, {384,49478}, {448,55891}, + {512,62288}, {640,75878}, {768,88700}, {896,102236}, + {1024,115102}, {1280,141441}, {1536,167820}, {1792,194205}, + {2048,220596}, {2560,273372}, {3072,326212}, {3584,379043}, + {4096,431915}, {5120,537472}, {6144,643342}, {7168,749114}, + {8192,854612}, {10240,1065905}, {12288,1277128}, {14336,1488599}, + {16384,1699794}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_4[] = { + {0,0}, {1,10302}, {2,10406}, {3,10514}, + {4,10608}, {5,10704}, {6,10796}, {7,10902}, + {8,11010}, {9,11098}, {10,11212}, {11,11304}, + {12,11404}, {13,11501}, {14,11590}, {15,11904}, + {16,11999}, {17,12099}, {18,12203}, {19,12301}, + {20,12402}, {21,12498}, {22,12602}, {23,12698}, + {24,12797}, {25,12901}, {26,12997}, {31,13498}, + {33,13694}, {36,13994}, {40,14396}, {48,15187}, + {56,15982}, {63,16685}, {64,16783}, {65,16876}, + {80,18377}, {96,19971}, {112,21564}, {127,23058}, + {128,23161}, {129,23260}, {160,26365}, {192,29551}, + {224,32749}, {256,35949}, {320,43034}, {384,49403}, + {448,55756}, {512,62161}, {640,75704}, {768,88497}, + {896,102015}, {1024,114819}, {1280,141163}, {1536,167471}, + {1792,193803}, {2048,220220}, {2560,272828}, {3072,325725}, + {3584,378498}, {4096,431144}, {5120,536712}, {6144,641979}, + {7168,747539}, {8192,852988}, {10240,1063897}, {12288,1275078}, + {14336,1485584}, {16384,1696059}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_5[] = { + {0,0}, {1,10305}, {2,10398}, {3,10505}, + {4,10605}, {5,10697}, {6,10807}, {7,10898}, + {8,10999}, {9,11097}, {11,11306}, {12,11403}, + {13,11502}, {14,11595}, {15,11894}, {16,12000}, + {17,12093}, {18,12194}, {19,12298}, {20,12400}, + {21,12499}, {22,12594}, {23,12688}, {25,12890}, + {26,12994}, {27,13093}, {28,13189}, {30,13395}, + {31,13489}, {32,13590}, {36,13983}, {40,14388}, + {48,15188}, {56,15984}, {63,16683}, {64,16776}, + {65,16872}, {80,18374}, {96,19968}, {112,21570}, + {127,23057}, {128,23165}, {129,23253}, {160,26358}, + {192,29545}, {224,32717}, {256,35912}, {320,42991}, + {384,49388}, {448,55793}, {512,62157}, {640,75692}, + {768,88483}, {896,102010}, {1024,114790}, {1280,141066}, + {1536,167373}, {1792,193776}, {2048,220049}, {2560,272855}, + {3072,325573}, {3584,378210}, {4096,430980}, {5120,536385}, + {6144,641781}, {7168,747192}, {8192,852597}, {10240,1063400}, + {12288,1274124}, {14336,1484884}, {16384,1695563}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_6[] = { + {0,0}, {2,2154}, {254,2154}, {256,4003}, + {258,4335}, {448,4335}, {512,4388}, {640,4671}, + {768,4836}, {896,4989}, {1024,5189}, {1280,5594}, + {1536,5992}, {1792,6392}, {2048,6795}, {2560,7693}, + {3072,8490}, {3584,9289}, {4096,10101}, {5120,11702}, + {6144,13296}, {7168,14929}, {8192,16534}, {10240,19752}, + {12288,22954}, {14336,26122}, {16384,29922}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_7[] = { + {0,0}, {2,2201}, {46,2202}, {56,2202}, + {60,2300}, {254,2300}, {256,5192}, {258,5207}, + {320,5321}, {384,5416}, {448,5518}, {512,5787}, + {640,5986}, {768,6195}, {896,6393}, {1024,6593}, + {1280,6982}, {1536,7395}, {2048,8193}, {2560,8892}, + {3072,9741}, {3584,10541}, {4096,11360}, {5120,12901}, + {6144,14506}, {7168,16150}, {8192,17744}, {10240,20951}, + {12288,24197}, {14336,27381}, {16384,31241}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_8[] = { + {0,0}, {2,5599}, {4,5599}, {6,5701}, + {8,5701}, {10,5824}, {12,5829}, {14,5893}, + {16,5893}, {18,5967}, {20,5968}, {24,5968}, + {26,5971}, {28,5971}, {30,5984}, {32,5984}, + {34,6022}, {38,6023}, {40,6023}, {42,6034}, + {48,6034}, {50,6044}, {52,6044}, {54,6062}, + {56,6214}, {60,6509}, {62,6569}, {64,6569}, + {66,6679}, {112,6679}, {126,6811}, {128,8587}, + {130,8587}, {160,8598}, {192,8702}, {254,8702}, + {256,8789}, {258,8789}, {320,8895}, {384,9003}, + {448,9094}, {512,9190}, {640,9390}, {896,9786}, + {1024,9994}, {1280,10403}, {1536,11358}, {1792,11815}, + {2048,12477}, {2560,14274}, {3072,15587}, {3584,17319}, + {4096,21061}, {5120,24188}, {6144,25886}, {7168,28620}, + {8192,32952}, {10240,38242}, {12288,44166}, {14336,49301}, + {16384,56734}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_9[] = { + {0,0}, {2,8522}, {4,8620}, {6,8722}, + {10,8924}, {12,9019}, {14,9118}, {16,9222}, + {18,9316}, {20,9424}, {22,9524}, {24,9617}, + {26,9721}, {28,9822}, {30,10123}, {32,10215}, + {34,10318}, {36,10414}, {38,10517}, {40,10615}, + {42,10721}, {44,10824}, {46,10913}, {48,11020}, + {50,11118}, {52,11215}, {54,11309}, {56,11415}, + {60,11618}, {62,11719}, {64,11811}, {66,11928}, + {72,12220}, {80,12618}, {96,13413}, {112,14219}, + {126,14908}, {128,15015}, {130,15106}, {160,16605}, + {192,18216}, {224,19812}, {254,21306}, {256,21406}, + {258,21505}, {320,24598}, {384,27787}, {448,30977}, + {512,34178}, {640,41256}, {768,47629}, {896,54053}, + {1024,60408}, {1280,73966}, {1536,86831}, {1792,100326}, + {2048,113186}, {2560,139506}, {3072,165862}, {3584,192225}, + {4096,218634}, {5120,271501}, {6144,324250}, {7168,376977}, + {8192,429845}, {10240,535661}, {12288,641316}, {14336,746791}, + {16384,852420}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_10[] = { + {0,0}, {2,8479}, {4,8578}, {6,8690}, + {8,8783}, {10,8882}, {12,8977}, {14,9077}, + {18,9283}, {20,9384}, {22,9476}, {24,9575}, + {26,9677}, {28,9775}, {30,10078}, {32,10178}, + {34,10272}, {36,10375}, {38,10474}, {40,10570}, + {42,10674}, {44,10766}, {46,10873}, {48,10971}, + {50,11075}, {52,11172}, {54,11268}, {56,11366}, + {60,11573}, {62,11669}, {64,11768}, {66,11872}, + {72,12172}, {80,12565}, {96,13361}, {112,14162}, + {126,14851}, {128,14964}, {130,15053}, {160,16550}, + {192,18153}, {224,19752}, {254,21242}, {256,21336}, + {258,21436}, {320,24522}, {384,27715}, {448,30898}, + {512,34090}, {640,41143}, {768,47595}, {896,53957}, + {1024,60341}, {1280,73855}, {1536,86673}, {1792,100189}, + {2048,112973}, {2560,139269}, {3072,165646}, {3584,191903}, + {4096,218261}, {5120,271061}, {6144,323668}, {7168,376276}, + {8192,429437}, {10240,534805}, {12288,640044}, {14336,745580}, + {16384,850965}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_11[] = { + {0,0}, {2,8494}, {6,8696}, {8,8796}, + {12,8995}, {14,9092}, {16,9200}, {18,9290}, + {22,9497}, {26,9690}, {28,9791}, {30,10095}, + {32,10200}, {34,10293}, {36,10390}, {38,10482}, + {40,10592}, {42,10690}, {44,10795}, {46,10891}, + {48,10992}, {52,11186}, {54,11287}, {56,11391}, + {60,11591}, {62,11696}, {64,11783}, {66,11890}, + {72,12586}, {80,12979}, {96,13776}, {112,14575}, + {126,15270}, {128,15373}, {130,15472}, {160,16680}, + {192,18273}, {224,19860}, {254,21359}, {256,21451}, + {258,21553}, {320,24647}, {384,28135}, {448,31026}, + {512,34214}, {640,41309}, {768,47738}, {896,54159}, + {1024,60861}, {1280,74099}, {1536,86921}, {1792,100478}, + {2048,113303}, {2560,139602}, {3072,166051}, {3584,192454}, + {4096,219298}, {5120,271749}, {6144,324524}, {7168,377635}, + {8192,430144}, {10240,535960}, {12288,641280}, {14336,746954}, + {16384,852260}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_12[] = { + {0,0}, {4,2007}, {12,2008}, {28,2009}, + {252,2010}, {256,4392}, {448,4392}, {508,4593}, + {512,4593}, {516,4806}, {640,4806}, {768,4885}, + {896,4995}, {1280,5592}, {1536,5991}, {1792,6394}, + {2048,6784}, {2560,7689}, {3072,8486}, {3584,9288}, + {4096,10089}, {5120,11697}, {6144,13281}, {7168,14887}, + {8192,16508}, {10240,19706}, {12288,23699}, {14336,26896}, + {16384,30730}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_13[] = { + {0,0}, {4,2044}, {8,2047}, {36,2047}, + {40,2049}, {56,2049}, {60,2145}, {124,2145}, + {128,2148}, {224,2149}, {252,2149}, {256,5398}, + {260,5407}, {320,5536}, {384,5645}, {448,5725}, + {508,5726}, {512,5795}, {516,5797}, {640,5996}, + {896,6394}, {1024,6592}, {1536,7396}, {1792,7795}, + {2048,8196}, {2560,8900}, {3072,9695}, {3584,10494}, + {4096,11300}, {5120,12913}, {6144,14507}, {7168,16102}, + {8192,17700}, {10240,20956}, {12288,25005}, {14336,28188}, + {16384,31977}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_14[] = { + {0,0}, {4,5886}, {8,5886}, {12,6102}, + {16,6104}, {20,6254}, {40,6254}, {44,6291}, + {48,6291}, {52,6305}, {56,6502}, {60,6768}, + {64,6800}, {68,7011}, {80,7011}, {84,7028}, + {120,7028}, {124,7098}, {128,8884}, {160,8884}, + {192,8976}, {252,8976}, {256,9071}, {260,9071}, + {320,9174}, {384,9279}, {448,9392}, {508,9392}, + {512,9480}, {516,9481}, {640,9673}, {768,9869}, + {896,10069}, {1024,10267}, {1280,10676}, {1536,11632}, + {1792,11871}, {2048,12729}, {2560,14541}, {3072,15853}, + {3584,17752}, {4096,21290}, {5120,24407}, {6144,26114}, + {7168,28779}, {8192,33199}, {10240,38493}, {12288,44405}, + {14336,49511}, {16384,56876}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_15[] = { + {0,0}, {4,8524}, {8,8630}, {12,8724}, + {16,8828}, {20,8927}, {24,9021}, {28,9126}, + {36,9323}, {40,9419}, {44,9522}, {48,9615}, + {52,9720}, {56,9819}, {60,10116}, {64,10221}, + {68,10317}, {76,10522}, {80,10618}, {84,10716}, + {88,10817}, {92,10914}, {96,11017}, {100,11115}, + {104,11216}, {108,11313}, {112,11416}, {120,11608}, + {124,11712}, {128,11810}, {144,12211}, {160,12611}, + {192,13401}, {224,14203}, {252,14899}, {256,15004}, + {260,15102}, {320,16602}, {384,18227}, {448,19814}, + {508,21307}, {512,21401}, {516,21503}, {640,24597}, + {768,27791}, {896,30988}, {1024,34167}, {1280,41270}, + {1536,47660}, {1792,54059}, {2048,60533}, {2560,73957}, + {3072,86888}, {3584,100365}, {4096,113229}, {5120,139552}, + {6144,165735}, {7168,192162}, {8192,218467}, {10240,271046}, + {12288,323774}, {14336,376405}, {16384,429017}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_16[] = { + {0,0}, {4,8504}, {8,8602}, {12,8707}, + {16,8808}, {20,8902}, {24,9004}, {36,9304}, + {40,9401}, {44,9500}, {48,9604}, {52,9706}, + {56,9811}, {60,10103}, {64,10198}, {68,10310}, + {72,10400}, {76,10500}, {80,10604}, {88,10799}, + {96,10998}, {100,11105}, {104,11196}, {112,11396}, + {120,11600}, {124,11698}, {128,11801}, {132,11902}, + {144,12203}, {160,12595}, {224,14191}, {252,14897}, + {256,14987}, {260,15086}, {320,16604}, {384,18184}, + {448,19798}, {508,21295}, {512,21379}, {516,21482}, + {640,24571}, {768,27760}, {896,30957}, {1024,34149}, + {1280,41233}, {1536,47656}, {1792,54092}, {2048,60484}, + {2560,73989}, {3072,86649}, {3584,100123}, {4096,113041}, + {5120,139309}, {6144,165614}, {7168,192090}, {8192,218311}, + {10240,270950}, {12288,323621}, {14336,376640}, {16384,429276}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_17[] = { + {0,0}, {4,8499}, {8,8604}, {24,9000}, + {28,9095}, {32,9193}, {36,9298}, {40,9396}, + {44,9499}, {48,9596}, {52,9699}, {56,9793}, + {60,10094}, {68,10293}, {72,10486}, {76,10589}, + {80,10687}, {84,10790}, {88,10883}, {92,10984}, + {96,11076}, {100,11183}, {104,11284}, {112,11479}, + {124,11781}, {128,11886}, {132,11980}, {144,12579}, + {192,13790}, {224,14286}, {252,14982}, {256,15083}, + {260,15183}, {320,16676}, {384,18260}, {448,19859}, + {508,21346}, {512,21439}, {516,21542}, {640,24628}, + {768,27816}, {896,31052}, {1024,34542}, {1280,41326}, + {1536,47748}, {1792,54145}, {2048,60555}, {2560,74034}, + {3072,86956}, {3584,100460}, {4096,113661}, {5120,139591}, + {6144,165857}, {7168,192137}, {8192,218882}, {10240,270987}, + {12288,323938}, {14336,376629}, {16384,429201}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_18[] = { + {0,0}, {8,2065}, {248,2066}, {256,4223}, + {264,4467}, {448,4467}, {504,4707}, {512,4707}, + {520,4923}, {640,4923}, {768,5010}, {896,5161}, + {1016,5525}, {1024,5525}, {1032,5761}, {1280,5761}, + {1536,6156}, {1792,6553}, {2048,6945}, {2560,7861}, + {3072,8657}, {3584,9452}, {4096,10264}, {5120,11875}, + {6144,13472}, {7168,15090}, {8192,16694}, {10240,19908}, + {12288,23106}, {14336,26316}, {16384,30111}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_19[] = { + {0,0}, {8,2169}, {16,2172}, {56,2172}, + {64,2271}, {144,2272}, {248,2272}, {256,5166}, + {264,5167}, {288,5169}, {320,5283}, {384,5334}, + {504,5334}, {512,5563}, {520,5564}, {640,5764}, + {768,5965}, {896,6024}, {1016,6024}, {1024,6359}, + {1032,6359}, {1280,6757}, {1536,7169}, {1792,7569}, + {2048,7970}, {2560,8672}, {3072,9473}, {3584,10275}, + {4096,11082}, {5120,12688}, {6144,14291}, {7168,15905}, + {8192,17515}, {10240,20762}, {12288,23974}, {14336,27155}, + {16384,30970}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_20[] = { + {0,0}, {8,5751}, {16,5981}, {24,6123}, + {40,6123}, {48,6165}, {56,6413}, {64,6666}, + {72,6745}, {80,6745}, {88,6795}, {120,6795}, + {128,8746}, {144,8746}, {152,8751}, {184,8751}, + {192,8846}, {216,8847}, {224,8850}, {240,8853}, + {248,8853}, {256,8943}, {264,8950}, {288,8952}, + {320,9053}, {384,9149}, {448,9255}, {504,9255}, + {512,9348}, {520,9348}, {640,9537}, {768,9740}, + {896,9936}, {1016,10109}, {1024,10143}, {1032,10346}, + {1280,10546}, {1536,11322}, {1792,11764}, {2048,12823}, + {2560,14460}, {3072,15729}, {3584,17447}, {4096,21199}, + {5120,24324}, {6144,26035}, {7168,28709}, {8192,33091}, + {10240,38355}, {12288,44269}, {14336,49444}, {16384,56842}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_21[] = { + {0,0}, {8,8498}, {16,8599}, {24,8705}, + {32,8794}, {40,8906}, {48,8998}, {56,9096}, + {64,9196}, {72,9299}, {80,9401}, {88,9500}, + {96,9614}, {104,9695}, {112,9795}, {120,10092}, + {128,10192}, {136,10297}, {144,10398}, {152,10498}, + {160,10588}, {168,10698}, {184,10902}, {192,10999}, + {200,11092}, {208,11194}, {216,11293}, {224,11399}, + {240,11589}, {248,11697}, {256,11796}, {264,11907}, + {288,12204}, {320,12592}, {384,13397}, {448,14191}, + {512,14991}, {520,15090}, {640,16592}, {768,18193}, + {896,19792}, {1016,21294}, {1024,21397}, {1032,21493}, + {1280,24565}, {1536,27706}, {1792,30897}, {2048,34125}, + {2560,41129}, {3072,47584}, {3584,53985}, {4096,60373}, + {5120,73895}, {6144,86650}, {7168,100173}, {8192,113034}, + {10240,139305}, {12288,165641}, {14336,191923}, {16384,218268}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_22[] = { + {0,0}, {8,8522}, {24,8721}, {32,8827}, + {40,8924}, {48,9023}, {56,9120}, {64,9220}, + {72,9312}, {80,9420}, {88,9518}, {96,9614}, + {104,9712}, {112,9821}, {120,10119}, {128,10218}, + {136,10324}, {152,10523}, {160,10615}, {168,10722}, + {176,10824}, {184,10931}, {192,11033}, {200,11122}, + {208,11233}, {224,11438}, {240,11634}, {248,11735}, + {256,11826}, {264,11932}, {288,12238}, {320,12638}, + {384,13433}, {448,14230}, {504,14925}, {512,15024}, + {520,15128}, {640,16638}, {768,18244}, {896,19836}, + {1016,21344}, {1024,21441}, {1032,21536}, {1280,24613}, + {1536,27848}, {1792,31032}, {2048,34228}, {2560,41319}, + {3072,47715}, {3584,54072}, {4096,60484}, {5120,74020}, + {6144,86819}, {7168,100391}, {8192,113241}, {10240,139572}, + {12288,165899}, {14336,192230}, {16384,219043}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_23[] = { + {0,0}, {8,8524}, {16,8613}, {24,8710}, + {32,8809}, {40,8910}, {48,9009}, {56,9125}, + {64,9211}, {72,9306}, {80,9406}, {88,9512}, + {96,9621}, {104,9704}, {112,9815}, {120,10199}, + {128,10206}, {136,10314}, {144,10408}, {152,10526}, + {168,10715}, {176,10819}, {184,10927}, {192,11011}, + {200,11123}, {208,11214}, {216,11317}, {224,11412}, + {240,11612}, {248,11710}, {256,11819}, {264,11914}, + {288,12214}, {384,13402}, {448,14207}, {504,14902}, + {512,15007}, {520,15105}, {640,16602}, {768,18204}, + {896,19805}, {1016,21304}, {1024,21408}, {1032,21505}, + {1280,24616}, {1536,27786}, {1792,30979}, {2048,34178}, + {2560,41273}, {3072,47727}, {3584,54201}, {4096,60627}, + {5120,74133}, {6144,87045}, {7168,100632}, {8192,113590}, + {10240,139958}, {12288,166342}, {14336,192740}, {16384,219049}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_24[] = { + {0,0}, {1,542}, {7,542}, {8,567}, + {26,567}, {27,571}, {127,571}, {128,3556}, + {640,3556}, {768,3742}, {1024,4142}, {1280,4740}, + {1536,4938}, {1792,5338}, {2048,5637}, {2560,6631}, + {3072,7626}, {3584,8227}, {4096,9224}, {5120,10722}, + {6144,12417}, {7168,14012}, {8192,15523}, {10240,18718}, + {12288,22726}, {14336,25912}, {16384,28842}, {20480,35232}, + {24576,41609}, {28672,47994}, {32768,55162}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_25[] = { + {0,0}, {1,622}, {2,652}, {21,652}, + {22,670}, {23,670}, {24,681}, {56,681}, + {63,740}, {65,740}, {80,789}, {127,789}, + {128,3802}, {160,3802}, {192,3902}, {224,3903}, + {256,4104}, {320,4199}, {384,4199}, {448,4289}, + {512,4299}, {640,4487}, {768,5084}, {896,5185}, + {1024,5285}, {1280,5698}, {1536,5885}, {1792,6684}, + {2048,6904}, {2560,7589}, {3072,8585}, {3584,9185}, + {4096,10101}, {5120,11805}, {6144,13413}, {7168,15014}, + {8192,16613}, {10240,19831}, {12288,23893}, {14336,27094}, + {16384,29901}, {20480,36314}, {24576,42685}, {28672,49090}, + {32768,56125}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_26[] = { + {0,0}, {1,648}, {2,651}, {5,651}, + {6,659}, {7,659}, {8,667}, {27,667}, + {28,671}, {31,671}, {32,687}, {33,734}, + {40,734}, {48,743}, {65,743}, {80,760}, + {96,760}, {112,786}, {127,786}, {128,3790}, + {160,3790}, {192,3888}, {224,3888}, {256,4093}, + {320,4197}, {384,4201}, {448,4280}, {512,4280}, + {640,4478}, {768,5084}, {896,5178}, {1024,5291}, + {1280,5679}, {1536,5880}, {1792,6680}, {2048,6882}, + {2560,7582}, {3072,8581}, {3584,9177}, {4096,10085}, + {5120,11793}, {6144,13413}, {7168,15021}, {8192,16616}, + {10240,19823}, {12288,23878}, {14336,27077}, {16384,29844}, + {20480,36260}, {24576,42672}, {28672,49077}, {32768,56161}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_27[] = { + {0,0}, {1,10057}, {2,10224}, {3,10325}, + {4,10423}, {5,10524}, {6,10620}, {7,10721}, + {8,10826}, {9,10919}, {10,11011}, {11,11125}, + {12,11195}, {13,11340}, {14,11414}, {15,11724}, + {17,11921}, {18,12023}, {20,12224}, {21,12322}, + {22,12419}, {24,12634}, {25,12720}, {26,12813}, + {27,12928}, {28,13036}, {30,13215}, {31,13334}, + {32,13423}, {33,13521}, {40,14228}, {48,15018}, + {56,15820}, {63,16513}, {64,16628}, {65,16719}, + {80,18231}, {96,19830}, {112,21445}, {127,22977}, + {128,23052}, {129,23141}, {160,26256}, {192,29457}, + {224,32657}, {256,35871}, {320,42472}, {384,48877}, + {448,55284}, {512,61701}, {640,74796}, {768,87570}, + {896,100642}, {1024,113495}, {1280,139296}, {1536,165156}, + {1792,191049}, {2048,216943}, {2560,268612}, {3072,320349}, + {3584,371936}, {4096,423762}, {5120,527397}, {6144,630805}, + {7168,734233}, {8192,837783}, {10240,1045026}, {12288,1252484}, + {14336,1458908}, {16384,1665863}, {20480,2080238}, {24576,2494174}, + {28672,2908097}, {32768,3321957}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_28[] = { + {0,0}, {1,10075}, {2,10197}, {3,10301}, + {4,10402}, {5,10502}, {6,10609}, {7,10702}, + {8,10802}, {9,10897}, {10,10998}, {11,11083}, + {12,11197}, {13,11284}, {14,11375}, {15,11695}, + {16,11804}, {17,11900}, {18,12003}, {19,12100}, + {20,12199}, {21,12296}, {22,12400}, {23,12494}, + {24,12594}, {25,12696}, {26,12812}, {27,12877}, + {28,13010}, {30,13202}, {31,13285}, {32,13397}, + {33,13498}, {36,13795}, {40,14149}, {48,14996}, + {56,15825}, {63,16483}, {64,16613}, {65,16699}, + {80,18208}, {96,19798}, {112,21405}, {127,22888}, + {128,23009}, {129,23107}, {160,26211}, {192,29409}, + {224,32599}, {256,35795}, {320,42394}, {384,48789}, + {448,55192}, {512,61590}, {640,74589}, {768,87432}, + {896,100426}, {1024,113292}, {1280,139051}, {1536,164845}, + {1792,190635}, {2048,216526}, {2560,268124}, {3072,319861}, + {3584,371367}, {4096,423112}, {5120,526462}, {6144,629686}, + {7168,733060}, {8192,836658}, {10240,1042878}, {12288,1250008}, + {14336,1456665}, {16384,1664023}, {20480,2076507}, {24576,2491260}, + {28672,2904375}, {32768,3316349}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_29[] = { + {0,0}, {1,10089}, {2,10207}, {3,10301}, + {4,10405}, {5,10504}, {7,10698}, {9,10899}, + {10,10983}, {11,11119}, {12,11192}, {13,11291}, + {14,11417}, {15,11698}, {16,11802}, {18,12000}, + {19,12096}, {20,12198}, {21,12298}, {22,12395}, + {24,12605}, {25,12714}, {26,12787}, {27,12910}, + {28,13010}, {30,13197}, {31,13329}, {32,13401}, + {33,13498}, {36,13800}, {40,14188}, {48,14995}, + {56,15793}, {63,16501}, {64,16592}, {65,16696}, + {80,18191}, {96,19798}, {112,21397}, {127,22887}, + {128,23006}, {129,23106}, {160,26211}, {192,29410}, + {224,32606}, {256,35805}, {320,42389}, {384,48791}, + {448,55197}, {512,61572}, {640,74626}, {768,87399}, + {896,100441}, {1024,113274}, {1280,139058}, {1536,164839}, + {1792,190736}, {2048,216548}, {2560,268110}, {3072,319767}, + {3584,371365}, {4096,423010}, {5120,526400}, {6144,629547}, + {7168,732994}, {8192,836239}, {10240,1042988}, {12288,1249754}, + {14336,1456154}, {16384,1663029}, {20480,2077570}, {24576,2489258}, + {28672,2903666}, {32768,3316914}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_30[] = { + {0,0}, {2,1636}, {20,1636}, {22,1660}, + {64,1660}, {66,1680}, {96,1680}, {112,1719}, + {126,1725}, {128,4031}, {130,4203}, {192,4203}, + {224,4276}, {256,4276}, {258,4350}, {512,4350}, + {640,4429}, {768,4727}, {896,4828}, {1024,5028}, + {1280,5724}, {1536,5925}, {1792,6275}, {2048,6624}, + {2560,7622}, {3072,8621}, {3584,9221}, {4096,10120}, + {5120,11738}, {6144,13336}, {8192,16528}, {10240,19729}, + {12288,23773}, {14336,26976}, {16384,29869}, {20480,36266}, + {24576,42697}, {28672,49107}, {32768,56299}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_31[] = { + {0,0}, {2,1727}, {4,1729}, {56,1729}, + {60,1829}, {126,1830}, {128,4723}, {130,4729}, + {160,4737}, {192,4836}, {224,4837}, {254,4837}, + {256,5035}, {258,5038}, {320,5140}, {384,5144}, + {448,5224}, {512,5224}, {640,5427}, {768,6018}, + {896,6125}, {1024,6219}, {1280,6626}, {1536,6927}, + {1792,7624}, {2048,7824}, {2560,8524}, {3072,9521}, + {3584,10220}, {4096,11211}, {5120,12765}, {6144,14362}, + {7168,15996}, {8192,17548}, {10240,20699}, {12288,24857}, + {14336,28021}, {16384,30757}, {20480,37216}, {28672,50000}, + {32768,56947}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_32[] = { + {0,0}, {2,1741}, {32,1741}, {34,1832}, + {62,1832}, {64,1840}, {66,1841}, {112,1842}, + {126,1842}, {128,4675}, {160,4675}, {192,4742}, + {224,4742}, {254,4846}, {256,5039}, {258,5039}, + {320,5139}, {384,5139}, {448,5233}, {512,5233}, + {640,5434}, {768,6032}, {896,6129}, {1024,6230}, + {1280,6629}, {1536,6937}, {1792,7629}, {2048,7828}, + {2560,8526}, {3072,9527}, {3584,10127}, {4096,12778}, + {5120,14376}, {6144,16102}, {7168,17679}, {8192,21361}, + {10240,25421}, {12288,30830}, {14336,34024}, {16384,39133}, + {20480,47971}, {24576,56570}, {28672,65287}, {32768,74010}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_33[] = { + {0,0}, {2,8403}, {4,8498}, {6,8602}, + {8,8696}, {10,8802}, {18,9197}, {20,9302}, + {22,9401}, {24,9492}, {26,9598}, {28,9699}, + {30,10000}, {34,10195}, {36,10296}, {38,10399}, + {40,10498}, {42,10601}, {48,10900}, {50,10996}, + {52,11100}, {54,11197}, {56,11297}, {60,11492}, + {62,11596}, {72,12095}, {80,12496}, {96,13294}, + {112,14095}, {126,14791}, {128,14894}, {130,14995}, + {160,16504}, {192,18115}, {224,19716}, {254,21213}, + {256,21314}, {258,21409}, {320,24512}, {384,27721}, + {448,30925}, {512,34115}, {640,40725}, {768,47100}, + {896,53503}, {1024,59900}, {1280,72902}, {1536,85713}, + {1792,98730}, {2048,111561}, {2560,137486}, {3072,163310}, + {3584,189193}, {4096,214983}, {6144,318335}, {7168,370023}, + {8192,422036}, {10240,525461}, {12288,628916}, {14336,732312}, + {16384,835717}, {20480,1042630}, {24576,1249444}, {28672,1456377}, + {32768,1663133}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_34[] = { + {0,0}, {2,8399}, {6,8602}, {8,8700}, + {10,8801}, {14,8996}, {16,9095}, {18,9198}, + {20,9292}, {22,9392}, {24,9497}, {26,9597}, + {28,9693}, {30,9992}, {32,10097}, {36,10296}, + {38,10397}, {40,10502}, {42,10595}, {44,10696}, + {48,10897}, {50,11000}, {52,11092}, {54,11199}, + {56,11299}, {60,11497}, {62,11602}, {64,11692}, + {66,11793}, {72,12093}, {96,13287}, {112,14092}, + {126,14794}, {128,14886}, {130,14985}, {160,16498}, + {192,18094}, {224,19704}, {254,21205}, {256,21309}, + {258,21405}, {320,24498}, {384,27702}, {448,30905}, + {512,34102}, {640,40670}, {768,47110}, {896,53461}, + {1024,59856}, {1280,72846}, {1536,85685}, {1792,98689}, + {2048,111548}, {2560,137377}, {3072,163168}, {3584,188993}, + {4096,214795}, {5120,266428}, {6144,318040}, {7168,369674}, + {8192,421298}, {10240,524714}, {12288,628044}, {14336,731658}, + {16384,834762}, {20480,1041900}, {24576,1247960}, {28672,1455296}, + {32768,1661373}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_35[] = { + {0,0}, {2,8419}, {4,8515}, {6,8610}, + {12,8914}, {14,9012}, {18,9213}, {20,9319}, + {22,9408}, {24,9514}, {26,9613}, {28,9720}, + {30,10010}, {32,10116}, {34,10206}, {36,10316}, + {38,10409}, {40,10514}, {42,10613}, {44,10710}, + {46,10810}, {48,10914}, {50,11012}, {52,11118}, + {54,11216}, {56,11310}, {60,11515}, {62,11611}, + {64,11706}, {66,11810}, {80,12516}, {96,13313}, + {112,14117}, {126,14815}, {128,14928}, {130,15027}, + {160,16520}, {192,18125}, {224,19729}, {254,21222}, + {256,21325}, {258,21423}, {320,24529}, {384,27729}, + {448,30932}, {512,34136}, {640,40740}, {768,47108}, + {896,53512}, {1024,59915}, {1280,72930}, {1536,85805}, + {1792,98825}, {2048,111649}, {2560,137503}, {3072,163321}, + {3584,189171}, {4096,215002}, {5120,266638}, {6144,318372}, + {7168,370198}, {8192,421862}, {10240,525320}, {12288,628941}, + {14336,732319}, {16384,836029}, {20480,1042839}, {24576,1249510}, + {28672,1456221}, {32768,1664069}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_36[] = { + {0,0}, {4,1677}, {124,1677}, {128,4089}, + {132,4263}, {192,4263}, {224,4321}, {256,4321}, + {260,4400}, {448,4400}, {508,4571}, {512,4571}, + {516,4637}, {640,4637}, {768,4771}, {896,4871}, + {1024,5069}, {1280,5770}, {1536,5965}, {1792,6319}, + {2048,6670}, {2560,7663}, {3072,8661}, {3584,9266}, + {4096,10165}, {5120,11765}, {6144,13368}, {7168,14987}, + {8192,16590}, {10240,19791}, {12288,22985}, {14336,26190}, + {16384,29100}, {20480,35493}, {24576,41900}, {28672,48309}, + {32768,55520}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_37[] = { + {0,0}, {4,1780}, {32,1781}, {56,1781}, + {60,1875}, {120,1875}, {124,1879}, {128,4785}, + {144,4785}, {160,4789}, {192,4885}, {252,4885}, + {256,5084}, {260,5086}, {320,5189}, {384,5190}, + {448,5273}, {508,5273}, {512,5276}, {516,5276}, + {640,5469}, {768,6075}, {896,6171}, {1024,6269}, + {1280,6675}, {1536,6967}, {1792,7666}, {2048,7868}, + {2560,8567}, {3072,9567}, {3584,10266}, {4096,11064}, + {5120,12811}, {6144,14364}, {7168,15968}, {8192,17650}, + {10240,20809}, {12288,24093}, {14336,27204}, {16384,29996}, + {20480,36382}, {24576,42802}, {28672,49191}, {32768,56219}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_38[] = { + {0,0}, {4,1799}, {20,1800}, {32,1800}, + {36,1890}, {60,1890}, {64,1894}, {68,1895}, + {88,1896}, {104,1897}, {124,1897}, {128,4751}, + {160,4751}, {192,4801}, {224,4801}, {252,4901}, + {256,5101}, {260,5101}, {320,5198}, {384,5198}, + {448,5286}, {512,5286}, {516,5395}, {640,5487}, + {768,6093}, {896,6185}, {1024,6285}, {1280,6686}, + {1536,6986}, {1792,7685}, {2048,7887}, {2560,8587}, + {3072,9585}, {3584,10186}, {4096,12837}, {5120,14551}, + {6144,16152}, {7168,17716}, {8192,21419}, {10240,24620}, + {12288,30021}, {14336,33336}, {16384,38434}, {20480,47051}, + {24576,55742}, {28672,64472}, {32768,73232}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_39[] = { + {0,0}, {4,8521}, {8,8625}, {12,8725}, + {16,8828}, {20,8919}, {24,9026}, {28,9125}, + {36,9318}, {40,9420}, {44,9525}, {48,9622}, + {52,9722}, {56,9815}, {60,10117}, {76,10516}, + {84,10724}, {88,10823}, {92,10918}, {96,11018}, + {100,11121}, {104,11220}, {108,11326}, {112,11420}, + {120,11624}, {132,11925}, {144,12227}, {160,12620}, + {192,13426}, {224,14221}, {252,14923}, {256,15021}, + {260,15122}, {320,16630}, {384,18246}, {448,19854}, + {508,21342}, {512,21431}, {516,21556}, {640,24647}, + {768,27862}, {896,31062}, {1024,34266}, {1280,40880}, + {1536,47302}, {1792,53709}, {2048,60125}, {2560,73132}, + {3072,85990}, {3584,99015}, {4096,111811}, {5120,137517}, + {6144,163347}, {7168,189205}, {8192,215035}, {10240,266715}, + {12288,318419}, {14336,369992}, {16384,421559}, {20480,524783}, + {24576,628114}, {28672,731967}, {32768,835101}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_40[] = { + {0,0}, {4,8536}, {8,8627}, {12,8725}, + {20,8932}, {24,9027}, {28,9135}, {32,9232}, + {36,9334}, {40,9425}, {44,9533}, {48,9631}, + {52,9728}, {56,9829}, {60,10120}, {64,10228}, + {68,10330}, {72,10427}, {76,10530}, {80,10630}, + {84,10725}, {88,10827}, {92,10928}, {96,11035}, + {100,11127}, {104,11226}, {108,11331}, {112,11431}, + {120,11623}, {124,11729}, {128,11824}, {132,11929}, + {144,12223}, {160,12625}, {192,13423}, {224,14229}, + {252,14922}, {256,15020}, {260,15127}, {320,16624}, + {384,18218}, {448,19814}, {508,21313}, {512,21428}, + {516,21511}, {640,24635}, {768,27846}, {896,31055}, + {1024,34257}, {1280,40863}, {1536,47274}, {1792,53690}, + {2048,60093}, {2560,73114}, {3072,85919}, {3584,98770}, + {4096,111748}, {5120,137426}, {6144,163330}, {7168,189112}, + {8192,214878}, {10240,266476}, {12288,318179}, {14336,369711}, + {16384,421390}, {20480,524392}, {24576,628086}, {28672,730782}, + {32768,834688}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_41[] = { + {0,0}, {4,8521}, {8,8618}, {16,8819}, + {20,8922}, {24,9017}, {28,9120}, {32,9221}, + {36,9318}, {40,9417}, {44,9524}, {48,9622}, + {52,9717}, {56,9822}, {60,10118}, {64,10222}, + {68,10325}, {72,10417}, {76,10524}, {80,10620}, + {84,10718}, {88,10815}, {92,10922}, {96,11019}, + {100,11122}, {108,11318}, {112,11423}, {120,11621}, + {124,11715}, {128,11823}, {132,11917}, {144,12216}, + {160,12617}, {192,13420}, {224,14217}, {252,14910}, + {256,15017}, {320,16618}, {384,18233}, {448,19816}, + {508,21325}, {512,21418}, {516,21525}, {640,24624}, + {768,27854}, {896,31060}, {1024,34272}, {1280,40892}, + {1536,47302}, {1792,53702}, {2048,60132}, {2560,73136}, + {3072,85838}, {3584,98883}, {4096,111685}, {5120,137399}, + {6144,163236}, {7168,189207}, {8192,215042}, {10240,266607}, + {12288,318338}, {14336,369897}, {16384,421616}, {20480,524800}, + {24576,628148}, {28672,731393}, {32768,834737}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_42[] = { + {0,0}, {8,1616}, {120,1616}, {128,4054}, + {136,4243}, {192,4243}, {200,4326}, {256,4326}, + {264,4408}, {448,4408}, {504,4540}, {512,4540}, + {520,4604}, {640,4604}, {768,4738}, {896,4838}, + {1016,5389}, {1024,5389}, {1032,5738}, {1280,5738}, + {1536,5934}, {1792,6287}, {2048,6639}, {2560,7638}, + {3072,8638}, {3584,9238}, {4096,10136}, {5120,11739}, + {6144,13336}, {7168,14938}, {8192,16542}, {10240,19768}, + {12288,22970}, {14336,26141}, {16384,29041}, {20480,35483}, + {24576,41884}, {28672,48288}, {32768,55539}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_43[] = { + {0,0}, {8,1742}, {48,1743}, {56,1743}, + {64,1828}, {120,1828}, {128,4721}, {184,4721}, + {192,4817}, {248,4817}, {256,5043}, {264,5045}, + {288,5048}, {320,5105}, {504,5105}, {512,5231}, + {520,5231}, {640,5431}, {768,5918}, {1016,5918}, + {1024,6229}, {1032,6229}, {1280,6629}, {1536,6928}, + {1792,7627}, {2048,7835}, {2560,8521}, {3072,9528}, + {3584,10221}, {4096,11229}, {5120,12768}, {6144,14319}, + {7168,16013}, {8192,17569}, {10240,20719}, {12288,24015}, + {14336,27120}, {16384,29910}, {20480,36299}, {24576,42683}, + {28672,49146}, {32768,56144}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_44[] = { + {0,0}, {8,1746}, {32,1746}, {40,1829}, + {56,1829}, {64,1842}, {120,1842}, {128,4712}, + {160,4712}, {168,4750}, {184,4751}, {192,4810}, + {224,4810}, {240,4847}, {248,4850}, {256,4928}, + {288,4928}, {320,5146}, {384,5146}, {448,5235}, + {504,5240}, {520,5240}, {640,5434}, {768,6033}, + {896,6135}, {1016,6235}, {1032,6235}, {1280,6633}, + {1536,6935}, {1792,7634}, {2048,7847}, {2560,8533}, + {3072,9533}, {3584,10265}, {4096,12877}, {5120,14392}, + {6144,16094}, {7168,17667}, {8192,21379}, {10240,24584}, + {12288,29987}, {14336,33393}, {16384,38291}, {20480,46995}, + {24576,55708}, {28672,64413}, {32768,73121}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_45[] = { + {0,0}, {8,8423}, {24,8615}, {32,8716}, + {40,8819}, {48,8920}, {56,9020}, {64,9118}, + {72,9220}, {80,9315}, {88,9423}, {96,9515}, + {104,9621}, {112,9718}, {120,10018}, {128,10117}, + {136,10214}, {144,10315}, {152,10424}, {160,10526}, + {168,10618}, {176,10726}, {184,10823}, {200,11021}, + {208,11124}, {216,11222}, {224,11324}, {240,11515}, + {248,11615}, {256,11713}, {264,11760}, {288,12102}, + {320,12516}, {384,13319}, {448,14125}, {504,14823}, + {512,14921}, {520,15028}, {640,16534}, {768,18118}, + {896,19727}, {1016,21274}, {1024,21335}, {1032,21433}, + {1280,24569}, {1536,27704}, {1792,30902}, {2048,34107}, + {2560,40703}, {3584,53497}, {4096,59890}, {5120,72887}, + {6144,85684}, {7168,98684}, {8192,111481}, {10240,137276}, + {12288,163265}, {14336,188852}, {16384,214637}, {20480,266532}, + {24576,318215}, {28672,369776}, {32768,421452}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_46[] = { + {0,0}, {8,8428}, {16,8534}, {24,8636}, + {40,8827}, {48,8931}, {64,9133}, {72,9236}, + {80,9336}, {88,9432}, {96,9539}, {104,9626}, + {112,9736}, {120,10032}, {128,10126}, {136,10234}, + {144,10326}, {152,10431}, {160,10529}, {168,10631}, + {176,10738}, {184,10830}, {200,11033}, {208,11128}, + {216,11230}, {224,11334}, {240,11534}, {248,11625}, + {256,11731}, {264,11829}, {288,12133}, {320,12527}, + {384,13324}, {448,14126}, {504,14837}, {512,14925}, + {520,15032}, {640,16537}, {768,18122}, {896,19751}, + {1016,21234}, {1024,21374}, {1032,21434}, {1280,24579}, + {1536,27723}, {1792,30967}, {2048,34090}, {2560,40682}, + {3072,47077}, {3584,53462}, {4096,59856}, {5120,72832}, + {6144,85624}, {7168,98617}, {8192,111398}, {10240,137169}, + {12288,162978}, {14336,188708}, {16384,214756}, {20480,266308}, + {24576,317993}, {28672,369536}, {32768,421109}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_47[] = { + {0,0}, {8,8420}, {16,8518}, {24,8615}, + {32,8715}, {40,8817}, {56,9019}, {64,9118}, + {72,9221}, {80,9322}, {88,9418}, {96,9521}, + {104,9622}, {112,9725}, {120,10020}, {128,10123}, + {144,10320}, {152,10423}, {160,10520}, {168,10622}, + {176,10722}, {184,10821}, {192,10923}, {200,11024}, + {208,11121}, {216,11216}, {224,11315}, {240,11521}, + {248,11617}, {256,11716}, {264,11828}, {288,12122}, + {320,12520}, {384,13315}, {448,14114}, {504,14821}, + {512,14920}, {520,15015}, {640,16517}, {768,18135}, + {896,19730}, {1016,21268}, {1032,21436}, {1280,24576}, + {1536,27757}, {1792,30892}, {2048,34088}, {2560,40709}, + {3072,47071}, {3584,53468}, {4096,59866}, {5120,72850}, + {6144,85648}, {7168,98765}, {8192,111536}, {10240,137373}, + {12288,163196}, {14336,189002}, {16384,214787}, {20480,266432}, + {24576,318093}, {28672,369712}, {32768,421378}, +}; + +static const ClgRepBulkCurve clg_rep_bulk_curves[] = { + { { CLG_REP_OP_MOVS, 1u, 0u, 0u }, clg_rep_bulk_pts_0, 29u }, + { { CLG_REP_OP_MOVS, 1u, 0u, 1u }, clg_rep_bulk_pts_1, 34u }, + { { CLG_REP_OP_MOVS, 1u, 0u, 2u }, clg_rep_bulk_pts_2, 53u }, + { { CLG_REP_OP_MOVS, 1u, 1u, 0u }, clg_rep_bulk_pts_3, 65u }, + { { CLG_REP_OP_MOVS, 1u, 1u, 1u }, clg_rep_bulk_pts_4, 70u }, + { { CLG_REP_OP_MOVS, 1u, 1u, 2u }, clg_rep_bulk_pts_5, 71u }, + { { CLG_REP_OP_MOVS, 2u, 0u, 0u }, clg_rep_bulk_pts_6, 27u }, + { { CLG_REP_OP_MOVS, 2u, 0u, 1u }, clg_rep_bulk_pts_7, 31u }, + { { CLG_REP_OP_MOVS, 2u, 0u, 2u }, clg_rep_bulk_pts_8, 61u }, + { { CLG_REP_OP_MOVS, 2u, 1u, 0u }, clg_rep_bulk_pts_9, 69u }, + { { CLG_REP_OP_MOVS, 2u, 1u, 1u }, clg_rep_bulk_pts_10, 69u }, + { { CLG_REP_OP_MOVS, 2u, 1u, 2u }, clg_rep_bulk_pts_11, 65u }, + { { CLG_REP_OP_MOVS, 4u, 0u, 0u }, clg_rep_bulk_pts_12, 29u }, + { { CLG_REP_OP_MOVS, 4u, 0u, 1u }, clg_rep_bulk_pts_13, 37u }, + { { CLG_REP_OP_MOVS, 4u, 0u, 2u }, clg_rep_bulk_pts_14, 50u }, + { { CLG_REP_OP_MOVS, 4u, 1u, 0u }, clg_rep_bulk_pts_15, 63u }, + { { CLG_REP_OP_MOVS, 4u, 1u, 1u }, clg_rep_bulk_pts_16, 60u }, + { { CLG_REP_OP_MOVS, 4u, 1u, 2u }, clg_rep_bulk_pts_17, 59u }, + { { CLG_REP_OP_MOVS, 8u, 0u, 0u }, clg_rep_bulk_pts_18, 31u }, + { { CLG_REP_OP_MOVS, 8u, 0u, 1u }, clg_rep_bulk_pts_19, 37u }, + { { CLG_REP_OP_MOVS, 8u, 0u, 2u }, clg_rep_bulk_pts_20, 52u }, + { { CLG_REP_OP_MOVS, 8u, 1u, 0u }, clg_rep_bulk_pts_21, 60u }, + { { CLG_REP_OP_MOVS, 8u, 1u, 1u }, clg_rep_bulk_pts_22, 59u }, + { { CLG_REP_OP_MOVS, 8u, 1u, 2u }, clg_rep_bulk_pts_23, 60u }, + { { CLG_REP_OP_STOS, 1u, 0u, 0u }, clg_rep_bulk_pts_24, 31u }, + { { CLG_REP_OP_STOS, 1u, 0u, 1u }, clg_rep_bulk_pts_25, 45u }, + { { CLG_REP_OP_STOS, 1u, 0u, 2u }, clg_rep_bulk_pts_26, 52u }, + { { CLG_REP_OP_STOS, 1u, 1u, 0u }, clg_rep_bulk_pts_27, 74u }, + { { CLG_REP_OP_STOS, 1u, 1u, 1u }, clg_rep_bulk_pts_28, 78u }, + { { CLG_REP_OP_STOS, 1u, 1u, 2u }, clg_rep_bulk_pts_29, 74u }, + { { CLG_REP_OP_STOS, 2u, 0u, 0u }, clg_rep_bulk_pts_30, 39u }, + { { CLG_REP_OP_STOS, 2u, 0u, 1u }, clg_rep_bulk_pts_31, 41u }, + { { CLG_REP_OP_STOS, 2u, 0u, 2u }, clg_rep_bulk_pts_32, 44u }, + { { CLG_REP_OP_STOS, 2u, 1u, 0u }, clg_rep_bulk_pts_33, 65u }, + { { CLG_REP_OP_STOS, 2u, 1u, 1u }, clg_rep_bulk_pts_34, 69u }, + { { CLG_REP_OP_STOS, 2u, 1u, 2u }, clg_rep_bulk_pts_35, 70u }, + { { CLG_REP_OP_STOS, 4u, 0u, 0u }, clg_rep_bulk_pts_36, 37u }, + { { CLG_REP_OP_STOS, 4u, 0u, 1u }, clg_rep_bulk_pts_37, 44u }, + { { CLG_REP_OP_STOS, 4u, 0u, 2u }, clg_rep_bulk_pts_38, 47u }, + { { CLG_REP_OP_STOS, 4u, 1u, 0u }, clg_rep_bulk_pts_39, 63u }, + { { CLG_REP_OP_STOS, 4u, 1u, 1u }, clg_rep_bulk_pts_40, 69u }, + { { CLG_REP_OP_STOS, 4u, 1u, 2u }, clg_rep_bulk_pts_41, 67u }, + { { CLG_REP_OP_STOS, 8u, 0u, 0u }, clg_rep_bulk_pts_42, 39u }, + { { CLG_REP_OP_STOS, 8u, 0u, 1u }, clg_rep_bulk_pts_43, 42u }, + { { CLG_REP_OP_STOS, 8u, 0u, 2u }, clg_rep_bulk_pts_44, 47u }, + { { CLG_REP_OP_STOS, 8u, 1u, 0u }, clg_rep_bulk_pts_45, 63u }, + { { CLG_REP_OP_STOS, 8u, 1u, 1u }, clg_rep_bulk_pts_46, 63u }, + { { CLG_REP_OP_STOS, 8u, 1u, 2u }, clg_rep_bulk_pts_47, 63u }, +}; diff --git a/callgrind/sim.c b/callgrind/sim.c index db3f9bef18..3431cb3e8e 100644 --- a/callgrind/sim.c +++ b/callgrind/sim.c @@ -29,6 +29,7 @@ */ #include "global.h" +#include "rep_bulk.h" /* Notes: @@ -1181,8 +1182,10 @@ static void log_1I1Dr(InstrInfo* ii, Addr data_addr, Word data_size) inc_costs(IrRes, cost_Ir, CLG_(current_state).cost + fullOffset(EG_IR) ); - inc_costs(DrRes, cost_Dr, - CLG_(current_state).cost + fullOffset(EG_DR) ); + /* REP bulk model: keep data misses, the curve replaces L1-hit refs. */ + if (!ii->rep_bulk_op || DrRes != L1_Hit) + inc_costs(DrRes, cost_Dr, + CLG_(current_state).cost + fullOffset(EG_DR) ); } } @@ -1209,8 +1212,9 @@ static void log_0I1Dr(InstrInfo* ii, Addr data_addr, Word data_size) else cost_Dr = CLG_(cost_base) + ii->cost_offset + ii->eventset->offset[EG_DR]; - inc_costs(DrRes, cost_Dr, - CLG_(current_state).cost + fullOffset(EG_DR) ); + if (!ii->rep_bulk_op || DrRes != L1_Hit) + inc_costs(DrRes, cost_Dr, + CLG_(current_state).cost + fullOffset(EG_DR) ); } } @@ -1244,8 +1248,9 @@ static void log_1I1Dw(InstrInfo* ii, Addr data_addr, Word data_size) inc_costs(IrRes, cost_Ir, CLG_(current_state).cost + fullOffset(EG_IR) ); - inc_costs(DwRes, cost_Dw, - CLG_(current_state).cost + fullOffset(EG_DW) ); + if (!ii->rep_bulk_op || DwRes != L1_Hit) + inc_costs(DwRes, cost_Dw, + CLG_(current_state).cost + fullOffset(EG_DW) ); } } @@ -1269,13 +1274,77 @@ static void log_0I1Dw(InstrInfo* ii, Addr data_addr, Word data_size) else cost_Dw = CLG_(cost_base) + ii->cost_offset + ii->eventset->offset[EG_DW]; - inc_costs(DwRes, cost_Dw, - CLG_(current_state).cost + fullOffset(EG_DW) ); + if (!ii->rep_bulk_op || DwRes != L1_Hit) + inc_costs(DwRes, cost_Dw, + CLG_(current_state).cost + fullOffset(EG_DW) ); } } +/*------------------------------------------------------------*/ +/*--- REP bulk cost model helper ---*/ +/*------------------------------------------------------------*/ + +/* Alignment class from the pointers of the operation's first element (see + * rep_bulk.h). STOS and merged read-modify events pass src == dst. */ +static unsigned rep_align_class(Addr src, Addr dst) +{ + UInt so = (UInt)(src & 63); + UInt dofs = (UInt)(dst & 63); + + if (so == 0 && dofs == 0) + return CLG_REP_ALIGN_ALIGNED; + if (so == 8 && dofs == 8) + return CLG_REP_ALIGN_SAME_UNALIGN; + return CLG_REP_ALIGN_OTHER; +} + +/* Called once per executed element of a supported rep-prefixed bulk copy; + * `rem_after` is the repeat count after VEX decremented it. The curve delta + * is charged to both Ct and Cl, so the consumer's sqrt(Ct*Cl) recovers it. */ +static void log_rep_bulk(InstrInfo* ii, ULong rem_after, Addr src, Addr dst, + Word dflag) +{ + exec_state* es = &CLG_(current_state); + unsigned op = ii->rep_bulk_op; + unsigned width = ii->rep_elem_width; + ULong rem_before = rem_after + 1; + unsigned dir = ((Long)dflag < 0) ? CLG_REP_DIR_BACKWARD + : CLG_REP_DIR_FORWARD; + ULong delta; + ULong * cost_C, *global_C; + + /* The alignment class is latched from the operation's first element, so + * address changes between elements cannot switch curves. The next + * element continues iff it reports the count and destination expected + * after the previous one; the instruction site cannot be compared since + * VEX translates the first element as part of the preceding block. */ + if (es->rep_next_rem != rem_before || es->rep_next_dst != dst) + es->rep_align = rep_align_class(src, dst); + es->rep_next_rem = rem_before - 1; + es->rep_next_dst = dir == CLG_REP_DIR_BACKWARD ? dst - width : dst + width; + + if (!es->collect) + return; + + delta = clg_rep_bulk_marginal(op, width, dir, es->rep_align, + rem_before * (ULong)width, (ULong)width); + + if (es->nonskipped) + cost_C = es->nonskipped->skipped + fullOffset(EG_CYCLES); + else + cost_C = CLG_(cost_base) + ii->cost_offset + + ii->eventset->offset[EG_CYCLES]; + global_C = es->cost + fullOffset(EG_CYCLES); + + cost_C[0] += delta; + cost_C[1] += delta; + global_C[0] += delta; + global_C[1] += delta; +} + + /*------------------------------------------------------------*/ /*--- Cache configuration ---*/ /*------------------------------------------------------------*/ @@ -1290,6 +1359,30 @@ static void cachesim_post_clo_init(void) /* Cache configurations. */ cache_t I1c, D1c, LLc; + /* REP bulk cost model (on by default). It needs real cache simulation and + * the per-instruction cycle model it replaces, and is currently amd64-only + * because it reads the repeat counter and DF from the guest state. In any + * other mode it stays inert, so prior behavior is unchanged. */ + if (CLG_(clo).rep_bulk_cost) { + Bool usable = CLG_(clo).simulate_cache && CLG_(clo).cycle_estimation + && !clo_simulate_writeback && !clo_collect_cacheuse; +#if !defined(VGA_amd64) + usable = False; +#endif + if (!usable) { + CLG_(clo).rep_bulk_cost = False; + } else { + UInt bad = clg_rep_bulk_init(); + if (bad > 0) { + VG_(message)(Vg_UserMsg, + "--rep-bulk-cost: calibrated REP bulk table rejected " + "(%u problem(s)); keeping the per-element model.\n", + bad); + CLG_(clo).rep_bulk_cost = False; + } + } + } + /* Initialize access handlers */ if (!CLG_(clo).simulate_cache) { CLG_(cachesim).log_1I0D = 0; @@ -1367,6 +1460,11 @@ static void cachesim_post_clo_init(void) CLG_(cachesim).log_0I1Dr_name = "log_0I1Dr"; CLG_(cachesim).log_0I1Dw_name = "log_0I1Dw"; + if (CLG_(clo).rep_bulk_cost) { + CLG_(cachesim).log_rep_bulk = log_rep_bulk; + CLG_(cachesim).log_rep_bulk_name = "log_rep_bulk"; + } + if (clo_collect_cacheuse) { /* Output warning for not supported option combinations */ diff --git a/callgrind/tests/Makefile.am b/callgrind/tests/Makefile.am index 0b7f9965db..105c3eb15a 100644 --- a/callgrind/tests/Makefile.am +++ b/callgrind/tests/Makefile.am @@ -44,9 +44,12 @@ EXTRA_DIST = \ find-source.vgtest find-source.stderr.exp find-source.post.exp \ inline-samefile.vgtest inline-samefile.stderr.exp inline-samefile.stdout.exp inline-samefile.post.exp \ inline-crossfile.vgtest inline-crossfile.stderr.exp inline-crossfile.stdout.exp inline-crossfile.post.exp \ - inline-crossfile-helper1.h inline-crossfile-helper2.h filter_inline + inline-crossfile-helper1.h inline-crossfile-helper2.h filter_inline \ + rep_bulk.c rep_bulk.vgtest rep_bulk_legacy.vgtest \ + rep_bulk.post.exp rep_bulk_legacy.post.exp rep_bulk_check.pl \ + rep_bulk.stderr.exp rep_bulk_legacy.stderr.exp -check_PROGRAMS = clreq register_desc register_desc_threads find_debuginfo simwork threads inline-samefile inline-crossfile runtime_obj_skip_c runtime_obj_skip_underflow deterministic_rng +check_PROGRAMS = clreq register_desc register_desc_threads find_debuginfo simwork threads inline-samefile inline-crossfile runtime_obj_skip_c runtime_obj_skip_underflow deterministic_rng rep_bulk if HAVE_PR_GET_AUXV check_PROGRAMS += prctl_get_auxv endif @@ -58,6 +61,10 @@ AM_CXXFLAGS += $(AM_FLAG_M3264_PRI) inline_samefile_CFLAGS = $(AM_CFLAGS) -O2 -g inline_crossfile_CFLAGS = $(AM_CFLAGS) -O2 -g +# Keep the measured part free of avoidable stack traffic: inline asm cases +# with the pointers already in registers. +rep_bulk_CFLAGS = $(AM_CFLAGS) -O2 + threads_LDADD = -lpthread deterministic_rng_LDADD = -lpthread register_desc_threads_LDADD = -lpthread diff --git a/callgrind/tests/rep_bulk.c b/callgrind/tests/rep_bulk.c new file mode 100644 index 0000000000..5a23462a85 --- /dev/null +++ b/callgrind/tests/rep_bulk.c @@ -0,0 +1,145 @@ +/*--------------------------------------------------------------------*/ +/*--- Regression test for the CodSpeed REP bulk cost model. ---*/ +/*--------------------------------------------------------------------*/ + +/* Each case is one callgrind part: ZERO_STATS, one rep instruction, + * DUMP_STATS. The post-processing script checks the emitted counters and + * that no new event names appear, with the cache geometry pinned by the + * .vgtest file (64 byte lines). */ + + +#if defined(__x86_64__) + +#include "../callgrind.h" + +#define COPY_SZ 10240u /* 160 x 64B lines */ +#define BACK_SZ 4096u +#define STOS_SZ 4096u +#define SELF_SZ 4096u +#define FLUSH_SZ (256u * 1024u) + +static unsigned char srcbuf[COPY_SZ + 128] __attribute__((aligned(64))); +static unsigned char dstbuf[COPY_SZ + 128] __attribute__((aligned(64))); +static volatile unsigned char flushbuf[FLUSH_SZ] __attribute__((aligned(64))); + +/* Evict the source/destination lines from D1. */ +static void flush_d1(void) +{ + unsigned i; + for (i = 0; i < FLUSH_SZ; i += 64) + flushbuf[i] = (unsigned char)i; +} + +static int verify_copy(void) +{ + unsigned i; + for (i = 0; i < COPY_SZ; i++) + if (dstbuf[i] != srcbuf[i]) + return 0; + return 1; +} + +/* Read every line of both copy buffers, so they are the most recently used + * lines in D1. The compiler may lower buffer initialization to a libc call + * whose first use runs lazy symbol binding in ld.so; those loads can evict + * buffer lines when the copy buffers already fill most ways of a set. Every + * load feeds the sink: Valgrind drops loads whose result is unused. */ +static volatile unsigned warm_sink; + +__attribute__((noinline)) +static void warm_buffers(void) +{ + unsigned i, acc = 0; + for (i = 0; i < COPY_SZ; i += 64) + acc += srcbuf[i] + dstbuf[i]; + warm_sink = acc; +} + +__attribute__((noinline)) +static void measure_copy(unsigned char *d, const unsigned char *s, unsigned long n) +{ + CALLGRIND_ZERO_STATS; + __asm__ __volatile__("rep movsb" + : "+D"(d), "+S"(s), "+c"(n) + : "a"(0u) + : "memory"); + CALLGRIND_DUMP_STATS; +} + +int main(void) +{ + unsigned char *d; + const unsigned char *s; + unsigned long n; + unsigned i; + + for (i = 0; i < COPY_SZ; i++) { + srcbuf[i] = (unsigned char)(i * 7u); + dstbuf[i] = 0; + } + /* Make both copy buffers D1-resident before the first measured part so + * the "hot" cases really have no misses. */ + warm_buffers(); + + /* part 0: one-byte forward copy (single element) */ + measure_copy(dstbuf, srcbuf, 1); + + /* part 1: hot 10 KiB forward copy, both buffers 64B-aligned */ + measure_copy(dstbuf, srcbuf, COPY_SZ); + if (!verify_copy()) + return 2; + + /* part 2: zero-length rep (no element may be charged) */ + measure_copy(dstbuf, srcbuf, 0); + if (!verify_copy()) + return 4; + + /* part 3: backward overlapping move (memmove pattern) */ + for (i = 0; i < BACK_SZ; i++) + dstbuf[i] = (unsigned char)(i * 13u + 1u); + d = dstbuf + 8 + BACK_SZ - 1; s = dstbuf + BACK_SZ - 1; n = BACK_SZ; + CALLGRIND_ZERO_STATS; + __asm__ __volatile__("std\n\trep movsb\n\tcld" + : "+D"(d), "+S"(s), "+c"(n) + : "a"(0u) + : "memory", "cc"); + CALLGRIND_DUMP_STATS; + for (i = 0; i < BACK_SZ; i++) + if (dstbuf[i + 8] != (unsigned char)(i * 13u + 1u)) + return 5; + + /* part 4: hot rep stosb */ + d = dstbuf; s = srcbuf; n = STOS_SZ; + CALLGRIND_ZERO_STATS; + __asm__ __volatile__("rep stosb" + : "+D"(d), "+S"(s), "+c"(n) + : "a"((unsigned char)0x5a) + : "memory"); + CALLGRIND_DUMP_STATS; + for (i = 0; i < STOS_SZ; i++) + if (dstbuf[i] != 0x5a) + return 6; + + /* part 5: self copy (source and destination are the same buffer) */ + measure_copy(dstbuf, dstbuf, SELF_SZ); + for (i = 0; i < SELF_SZ; i++) + if (dstbuf[i] != 0x5a) + return 7; + + /* part 6: cold 10 KiB forward copy after flushing D1 */ + flush_d1(); + measure_copy(dstbuf, srcbuf, COPY_SZ); + if (!verify_copy()) + return 3; + + return 0; +} + +#else /* !__x86_64__ */ + +int main(void) +{ + return 0; +} + +#endif diff --git a/callgrind/tests/rep_bulk.post.exp b/callgrind/tests/rep_bulk.post.exp new file mode 100644 index 0000000000..17913a8e18 --- /dev/null +++ b/callgrind/tests/rep_bulk.post.exp @@ -0,0 +1,10 @@ +events: ok +one_byte_bulk_cost: ok +hot_lines_suppressed: ok +instruction_refs_kept: ok +bulk_cycles: ok +backward: ok +stos: ok +self: ok +cold_miss_refs: ok +cache_independent_cycles: ok diff --git a/callgrind/tests/rep_bulk.stderr.exp b/callgrind/tests/rep_bulk.stderr.exp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/callgrind/tests/rep_bulk.vgtest b/callgrind/tests/rep_bulk.vgtest new file mode 100644 index 0000000000..0445cc0825 --- /dev/null +++ b/callgrind/tests/rep_bulk.vgtest @@ -0,0 +1,5 @@ +prereq: rm -f callgrind.rep_bulk.out* && test "$(uname -m)" = x86_64 +prog: rep_bulk +vgopts: -q --combine-dumps=yes --cache-sim=yes --cycle-estimation=yes --I1=32768,8,64 --D1=32768,8,64 --LL=8388608,16,64 --callgrind-out-file=callgrind.rep_bulk.out +post: perl ./rep_bulk_check.pl callgrind.rep_bulk.out on +cleanup: rm -f callgrind.rep_bulk.out callgrind.rep_bulk.out.* diff --git a/callgrind/tests/rep_bulk_check.pl b/callgrind/tests/rep_bulk_check.pl new file mode 100755 index 0000000000..29edbc83ca --- /dev/null +++ b/callgrind/tests/rep_bulk_check.pl @@ -0,0 +1,122 @@ +#!/usr/bin/perl +# Check the callgrind output of the rep_bulk test. +# Usage: rep_bulk_check.pl on|off +use strict; +use warnings; + +my ($file, $mode) = @ARGV; +die "usage: rep_bulk_check.pl on|off\n" + unless defined $file && defined $mode && ($mode eq 'on' || $mode eq 'off'); + +open(my $fh, '<', $file) or die "cannot open $file: $!\n"; +my @events; +my @parts; +while (my $line = <$fh>) { + chomp $line; + if ($line =~ /^events:\s+(.*)$/) { + @events = split(' ', $1) unless @events; + } elsif ($line =~ /^summary:\s+(.*)$/) { + my @v = split(' ', $1); + my %p; + for my $i (0 .. $#events) { + $p{$events[$i]} = $v[$i]; + } + push @parts, \%p; + } +} +close($fh); + +my @fail; +sub check { + my ($name, $ok) = @_; + if ($ok) { + print "$name: ok\n"; + } else { + print "$name: MISMATCH\n"; + push @fail, $name; + } +} +sub val { + my ($part, $name) = @_; + die "part $part has no event $name\n" unless defined $parts[$part]{$name}; + return $parts[$part]{$name} + 0; +} + +my $expected_events = 'Ir Dr Dw I1mr D1mr D1mw ILmr DLmr DLmw Ct Cl'; +die "expected at least 7 dump parts, got ".scalar(@parts)."\n" if @parts < 7; + +check('events', join(' ', @events) eq $expected_events); + +# Deltas of a part against part 2 (zero-length rep: pure wrapper cost). +sub delta { my ($part, $name) = @_; return val($part, $name) - val(2, $name); } + +if ($mode eq 'on') { + check('one_byte_bulk_cost', val(0, 'Ct') > val(2, 'Ct')); + + check('hot_lines_suppressed', + delta(1, 'Dr') == 0 && delta(1, 'Dw') == 0 + && delta(1, 'D1mr') == 0 && delta(1, 'D1mw') == 0); + + # Instruction-side counters are never modeled: one element executes one + # more IMark than the zero-length dispatch (1 vs 2 dispatches). + check('instruction_refs_kept', + delta(0, 'Ir') == 1 && delta(1, 'Ir') == 10240); + + # The aligned curve prices 10 KiB at about 19K centicycles, the + # misaligned one at about 38K: the class must come from the first element. + my $ct = delta(1, 'Ct'); + check('bulk_cycles', $ct > 0 && $ct < 25000 && delta(1, 'Cl') > 0); + + # A 4 KiB backward copy costs more than the 10 KiB forward one only if + # every element keeps the backward curve. + check('backward', + delta(3, 'Ct') > delta(1, 'Ct') + && delta(3, 'Dr') == 0 && delta(3, 'Dw') == 0); + + check('stos', + val(4, 'Ct') > val(2, 'Ct') + && delta(4, 'Dr') == 0 && delta(4, 'Dw') == 0); + + check('self', + val(5, 'Ct') > val(2, 'Ct') + && delta(5, 'Dr') == 0 && delta(5, 'Dw') == 0); + + my $dr = delta(6, 'Dr'); + my $dw = delta(6, 'Dw'); + my $d1r = delta(6, 'D1mr'); + my $d1w = delta(6, 'D1mw'); + check('cold_miss_refs', + $dr >= 160 && $dr < 1024 && + $dw >= 160 && $dw < 1024 && + $d1r >= 160 && $d1r < 1024 && + $d1w >= 160 && $d1w < 1024); + + check('cache_independent_cycles', + val(1, 'Ct') == val(6, 'Ct') && val(1, 'Cl') == val(6, 'Cl')); +} else { + check('element_refs', delta(0, 'Dr') == 1 && delta(0, 'Dw') == 1); + + check('legacy_hot_refs', delta(1, 'Dr') == 10240 && delta(1, 'Dw') == 10240); + + check('legacy_cycles', delta(1, 'Ct') > 500000); + + check('legacy_zero_dispatch', delta(0, 'Ct') >= 100); + + check('legacy_backward_refs', delta(3, 'Dr') == 4096 && delta(3, 'Dw') == 4096); + + check('legacy_stos_refs', delta(4, 'Dr') == 0 && delta(4, 'Dw') == 4096); + + + my $dr = delta(6, 'Dr'); + my $d1r = delta(6, 'D1mr'); + my $d1w = delta(6, 'D1mw'); + check('legacy_cold_refs', + $dr >= 10240 && $d1r >= 160 && $d1r < 1024 && + $d1w >= 160 && $d1w < 1024); +} + +if (@fail) { + print "FAILED: ", join(', ', @fail), "\n"; + exit 1; +} +exit 0; diff --git a/callgrind/tests/rep_bulk_legacy.post.exp b/callgrind/tests/rep_bulk_legacy.post.exp new file mode 100644 index 0000000000..192e7303e5 --- /dev/null +++ b/callgrind/tests/rep_bulk_legacy.post.exp @@ -0,0 +1,8 @@ +events: ok +element_refs: ok +legacy_hot_refs: ok +legacy_cycles: ok +legacy_zero_dispatch: ok +legacy_backward_refs: ok +legacy_stos_refs: ok +legacy_cold_refs: ok diff --git a/callgrind/tests/rep_bulk_legacy.stderr.exp b/callgrind/tests/rep_bulk_legacy.stderr.exp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/callgrind/tests/rep_bulk_legacy.vgtest b/callgrind/tests/rep_bulk_legacy.vgtest new file mode 100644 index 0000000000..b052754ca8 --- /dev/null +++ b/callgrind/tests/rep_bulk_legacy.vgtest @@ -0,0 +1,5 @@ +prereq: rm -f callgrind.rep_bulk.out* && test "$(uname -m)" = x86_64 +prog: rep_bulk +vgopts: -q --combine-dumps=yes --cache-sim=yes --cycle-estimation=yes --rep-bulk-cost=no --I1=32768,8,64 --D1=32768,8,64 --LL=8388608,16,64 --callgrind-out-file=callgrind.rep_bulk.out +post: perl ./rep_bulk_check.pl callgrind.rep_bulk.out off +cleanup: rm -f callgrind.rep_bulk.out callgrind.rep_bulk.out.* diff --git a/callgrind/threads.c b/callgrind/threads.c index 642813e4cb..8f741afbc7 100644 --- a/callgrind/threads.c +++ b/callgrind/threads.c @@ -441,6 +441,9 @@ void CLG_(init_exec_state)(exec_state* es) es->jmps_passed = 0; es->bbcc = 0; es->nonskipped = 0; + es->rep_next_dst = 0; + es->rep_next_rem = 0; + es->rep_align = 0; } @@ -540,6 +543,9 @@ exec_state* exec_state_save(void) es->jmps_passed = CLG_(current_state).jmps_passed; es->bbcc = CLG_(current_state).bbcc; es->nonskipped = CLG_(current_state).nonskipped; + es->rep_next_dst = CLG_(current_state).rep_next_dst; + es->rep_next_rem = CLG_(current_state).rep_next_rem; + es->rep_align = CLG_(current_state).rep_align; CLG_ASSERT(es->cost == CLG_(current_state).cost); CLG_DEBUGIF(1) { @@ -565,6 +571,9 @@ exec_state* exec_state_restore(void) CLG_(current_state).jmps_passed = es->jmps_passed; CLG_(current_state).bbcc = es->bbcc; CLG_(current_state).nonskipped = es->nonskipped; + CLG_(current_state).rep_next_dst = es->rep_next_dst; + CLG_(current_state).rep_next_rem = es->rep_next_rem; + CLG_(current_state).rep_align = es->rep_align; CLG_(current_state).cost = es->cost; CLG_(current_state).sig = es->sig; From 69a748fcc9f7c257315d12ba6b0b3b0b14450352 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 5 Nov 2026 12:07:15 +0100 Subject: [PATCH 2/8] fixup! fix(callgrind): model REP copies with calibrated bulk costs Index the curve table directly instead of validating it at runtime. The generator already checks the table and emits curves in key order, so an assert at startup replaces the index build, key validation and the fallback message. --- callgrind/rep_bulk.c | 83 +++++++++----------------------------------- callgrind/rep_bulk.h | 7 ++-- callgrind/sim.c | 27 +++++--------- 3 files changed, 28 insertions(+), 89 deletions(-) diff --git a/callgrind/rep_bulk.c b/callgrind/rep_bulk.c index 0688381d83..e6e7a9d173 100644 --- a/callgrind/rep_bulk.c +++ b/callgrind/rep_bulk.c @@ -6,6 +6,7 @@ /* CodSpeed extension. See rep_bulk.h. */ #include "rep_bulk.h" +#include "pub_tool_libcassert.h" /* Generated by the calibration pipeline. Defines * clg_rep_bulk_provenance[], clg_rep_bulk_pts_[] and @@ -19,28 +20,14 @@ #define N_DIRS 2 #define N_ALIGN 3 -/* Exact-key index, resolved once at startup. The model is enabled only if - * every key is present. */ -static const ClgRepBulkCurve* curve_idx[CLG_REP_OP_STOS + 1][N_WIDTHS] - [N_DIRS][N_ALIGN]; - -static Int width_index(unsigned elem_width) +/* The generated table lists the curves in key order: op, width, direction, + * alignment class. */ +static const ClgRepBulkCurve* curve(unsigned op, unsigned elem_width, + unsigned direction, unsigned align_class) { - switch (elem_width) { - case 1: return 0; - case 2: return 1; - case 4: return 2; - case 8: return 3; - default: return -1; - } -} - -static Bool key_valid(const ClgRepBulkKey* k) -{ - return (k->op == CLG_REP_OP_MOVS || k->op == CLG_REP_OP_STOS) - && width_index(k->elem_width) >= 0 - && k->direction < N_DIRS - && k->align_class < N_ALIGN; + UInt w = (UInt)__builtin_ctz(elem_width); + return &clg_rep_bulk_curves[(((op - CLG_REP_OP_MOVS) * N_WIDTHS + w) + * N_DIRS + direction) * N_ALIGN + align_class]; } /* Interpolate linearly between the neighbouring sampled byte sizes. Integer @@ -77,63 +64,25 @@ static ULong cost_at(const ClgRepBulkCurve* c, ULong bytes) return p[lo].centicycles + (off * dcost) / dbytes; } -UInt clg_rep_bulk_init(void) +void clg_rep_bulk_init(void) { - UInt bad = 0, i, w, d, a; + UInt i; - /* Table validation: keys, F(0)==0, ascending sizes, monotone costs. */ + (void)clg_rep_bulk_provenance; + tl_assert(N_CURVES == 2 * N_WIDTHS * N_DIRS * N_ALIGN); for (i = 0; i < N_CURVES; i++) { const ClgRepBulkCurve* c = &clg_rep_bulk_curves[i]; - UInt j; - - if (!key_valid(&c->key) || c->npts < 2) { - bad++; - continue; - } - if (c->pts[0].bytes != 0 || c->pts[0].centicycles != 0) { - bad++; - continue; - } - for (j = 1; j < c->npts; j++) { - if (c->pts[j].bytes <= c->pts[j - 1].bytes - || c->pts[j].centicycles < c->pts[j - 1].centicycles) { - bad++; - break; - } - } - if (j != c->npts) - continue; - - curve_idx[c->key.op][width_index(c->key.elem_width)] - [c->key.direction][c->key.align_class] = c; + tl_assert(c->npts >= 2); + tl_assert(curve(c->key.op, c->key.elem_width, c->key.direction, + c->key.align_class) == c); } - - /* Every key is required: MOVS and STOS, all four widths, forward and - * backward, all three alignment classes. There is no cross-direction or - * cross-class substitution; the caller disables the model if any key is - * missing so the legacy per-element model stays in effect. */ - for (w = 0; w < N_WIDTHS; w++) - for (d = 0; d < N_DIRS; d++) - for (a = 0; a < N_ALIGN; a++) { - if (!curve_idx[CLG_REP_OP_MOVS][w][d][a]) - bad++; - if (!curve_idx[CLG_REP_OP_STOS][w][d][a]) - bad++; - } - - /* Keep the provenance string referenced (it documents the artifact). */ - if (clg_rep_bulk_provenance[0] == 0) - bad++; - - return bad; } ULong clg_rep_bulk_marginal(unsigned op, unsigned elem_width, unsigned direction, unsigned align_class, ULong remaining_bytes, ULong elem_bytes) { - const ClgRepBulkCurve* c = - curve_idx[op][width_index(elem_width)][direction][align_class]; + const ClgRepBulkCurve* c = curve(op, elem_width, direction, align_class); /* F is monotone, so the difference is never negative and telescopes to * F(total) over one operation. */ diff --git a/callgrind/rep_bulk.h b/callgrind/rep_bulk.h index b09b1a8e91..d565350f74 100644 --- a/callgrind/rep_bulk.h +++ b/callgrind/rep_bulk.h @@ -59,10 +59,9 @@ typedef struct { UInt npts; } ClgRepBulkCurve; -/* Build the lookup index and check the table. Returns the number of table - * problems (0 == clean); the caller reports them. Call once at startup; - * clg_rep_bulk_marginal is only valid after it returned 0. */ -UInt clg_rep_bulk_init(void); +/* Assert that the generated table has every key in index order. Call once + * at startup before clg_rep_bulk_marginal. */ +void clg_rep_bulk_init(void); /* Marginal bulk cost of the element at `remaining_bytes` (bytes still to * execute before this element, a nonzero multiple of elem_bytes), in diff --git a/callgrind/sim.c b/callgrind/sim.c index 3431cb3e8e..d7cb2d5add 100644 --- a/callgrind/sim.c +++ b/callgrind/sim.c @@ -1363,25 +1363,16 @@ static void cachesim_post_clo_init(void) * the per-instruction cycle model it replaces, and is currently amd64-only * because it reads the repeat counter and DF from the guest state. In any * other mode it stays inert, so prior behavior is unchanged. */ - if (CLG_(clo).rep_bulk_cost) { - Bool usable = CLG_(clo).simulate_cache && CLG_(clo).cycle_estimation - && !clo_simulate_writeback && !clo_collect_cacheuse; -#if !defined(VGA_amd64) - usable = False; +#if defined(VGA_amd64) + CLG_(clo).rep_bulk_cost = CLG_(clo).rep_bulk_cost + && CLG_(clo).simulate_cache + && CLG_(clo).cycle_estimation + && !clo_simulate_writeback && !clo_collect_cacheuse; +#else + CLG_(clo).rep_bulk_cost = False; #endif - if (!usable) { - CLG_(clo).rep_bulk_cost = False; - } else { - UInt bad = clg_rep_bulk_init(); - if (bad > 0) { - VG_(message)(Vg_UserMsg, - "--rep-bulk-cost: calibrated REP bulk table rejected " - "(%u problem(s)); keeping the per-element model.\n", - bad); - CLG_(clo).rep_bulk_cost = False; - } - } - } + if (CLG_(clo).rep_bulk_cost) + clg_rep_bulk_init(); /* Initialize access handlers */ if (!CLG_(clo).simulate_cache) { From b2bf875c253e844a94f446bacd2362cf4b1d21b6 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 5 Nov 2026 12:08:13 +0100 Subject: [PATCH 3/8] fixup! fix(callgrind): model REP copies with calibrated bulk costs Return the op from clg_rep_bulk_classify and write the element width straight into the InstrInfo, dropping ClgRepBulkInfo and the redundant length checks (the IMark length is already asserted). Repeated prefixes such as F3 F3 A4 are now accepted, matching how the CPU executes them. --- callgrind/main.c | 8 ++--- callgrind/rep_bulk.c | 84 ++++++++++++-------------------------------- callgrind/rep_bulk.h | 13 +++---- 3 files changed, 29 insertions(+), 76 deletions(-) diff --git a/callgrind/main.c b/callgrind/main.c index a9b2b572e3..1b4ebf0336 100644 --- a/callgrind/main.c +++ b/callgrind/main.c @@ -1130,11 +1130,9 @@ IRSB* CLG_(instrument)( VgCallbackClosure* closure, // Classify supported rep-prefixed bulk copies (rep movs/rep stos) for // the REP bulk cost model. The decision is static per instruction. if (CLG_(clo).rep_bulk_cost && !clgs.seen_before) { - ClgRepBulkInfo ri; - clg_rep_bulk_classify((const unsigned char*)cia, isize, &ri); - curr_inode->rep_bulk_op = ri.op; - curr_inode->rep_elem_width = ri.elem_width; - if (ri.op != CLG_REP_OP_NONE) { + curr_inode->rep_bulk_op = clg_rep_bulk_classify( + (const UChar*)cia, isize, &curr_inode->rep_elem_width); + if (curr_inode->rep_bulk_op != CLG_REP_OP_NONE) { /* The calibrated bulk curve replaces this instruction's * static per-element cycle estimate completely: drop it from * both the dump-time charge and the inclusive prefix sums. */ diff --git a/callgrind/rep_bulk.c b/callgrind/rep_bulk.c index e6e7a9d173..d9f1b5cf61 100644 --- a/callgrind/rep_bulk.c +++ b/callgrind/rep_bulk.c @@ -89,70 +89,30 @@ ULong clg_rep_bulk_marginal(unsigned op, unsigned elem_width, return cost_at(c, remaining_bytes) - cost_at(c, remaining_bytes - elem_bytes); } -/* Prefixes followed by A4/A5/AA/AB; REX must directly precede the opcode. */ -void clg_rep_bulk_classify(const unsigned char* bytes, unsigned len, - ClgRepBulkInfo* out) +/* Prefixes, then A4/A5 (MOVS) or AA/AB (STOS) as the last byte; REX must + * directly precede the opcode. */ +ClgRepBulkOp clg_rep_bulk_classify(const UChar* bytes, UInt len, + UChar* elem_width) { - out->op = CLG_REP_OP_NONE; - out->elem_width = 0; -#if defined(VGA_amd64) - unsigned i = 0; - int rep = 0, opsz16 = 0, addr32p = 0, rex_w = 0; - - if (!bytes || len < 2 || len > 15) - return; - - while (i < len) { - unsigned char c = bytes[i]; - if (c == 0xF3 && !rep) { - rep = 1; - i++; - continue; + UChar opc = bytes[len - 1]; + Bool rep = False, opsz16 = False, rex_w = False; + UInt i; + + for (i = 0; i + 1 < len; i++) { + switch (bytes[i]) { + case 0xF3: rep = True; break; + case 0x66: opsz16 = True; break; + case 0x67: break; + default: + if ((bytes[i] & 0xF0) != 0x40 || i + 2 != len) + return CLG_REP_OP_NONE; + rex_w = (bytes[i] & 0x08) != 0; } - if (c == 0xF2) - return; /* REPNE: never a supported bulk copy */ - if (c == 0x66 && !opsz16) { - opsz16 = 1; - i++; - continue; - } - if (c == 0x67 && !addr32p) { - addr32p = 1; - i++; - continue; - } - if (c >= 0x40 && c <= 0x4F) { - rex_w = (c & 0x08) != 0; - i++; - } - break; } - /* Exactly the opcode must follow the prefix run: no operands, no - * trailing bytes (the IMark length always covers the whole insn). */ - if (!rep || i + 1 != len) - return; - - switch (bytes[i]) { - case 0xA4: /* movsb */ - out->op = CLG_REP_OP_MOVS; - out->elem_width = 1; - break; - case 0xA5: /* movsw/movsd/movsq */ - out->op = CLG_REP_OP_MOVS; - out->elem_width = rex_w ? 8 : (opsz16 ? 2 : 4); - break; - case 0xAA: /* stosb */ - out->op = CLG_REP_OP_STOS; - out->elem_width = 1; - break; - case 0xAB: /* stosw/stosd/stosq */ - out->op = CLG_REP_OP_STOS; - out->elem_width = rex_w ? 8 : (opsz16 ? 2 : 4); - break; - } -#else - (void)bytes; - (void)len; -#endif + if (!rep || ((opc & ~1) != 0xA4 && (opc & ~1) != 0xAA)) + return CLG_REP_OP_NONE; + /* Even opcodes copy bytes; odd ones use the operand size. */ + *elem_width = (opc & 1) == 0 ? 1 : rex_w ? 8 : opsz16 ? 2 : 4; + return (opc & ~1) == 0xA4 ? CLG_REP_OP_MOVS : CLG_REP_OP_STOS; } diff --git a/callgrind/rep_bulk.h b/callgrind/rep_bulk.h index d565350f74..486804bb32 100644 --- a/callgrind/rep_bulk.h +++ b/callgrind/rep_bulk.h @@ -26,11 +26,6 @@ typedef enum { CLG_REP_OP_STOS = 2 /* rep stos{b,w,d,q}: one write per element */ } ClgRepBulkOp; -typedef struct { - unsigned char op; /* ClgRepBulkOp */ - unsigned char elem_width; /* element size in bytes (1,2,4,8), 0 if none */ -} ClgRepBulkInfo; - /* From the guest DF flag, which is constant during an instruction. */ #define CLG_REP_DIR_FORWARD 0 #define CLG_REP_DIR_BACKWARD 1 @@ -73,9 +68,9 @@ ULong clg_rep_bulk_marginal(unsigned op, unsigned elem_width, ULong remaining_bytes, ULong elem_bytes); /* Classify the amd64 guest instruction in [bytes, bytes+len) by its - * encoding (no Capstone needed). Always initialises *out; out->op is - * CLG_REP_OP_NONE unless it is a rep-prefixed MOVS or STOS. */ -void clg_rep_bulk_classify(const unsigned char* bytes, unsigned len, - ClgRepBulkInfo* out); + * encoding. Returns CLG_REP_OP_NONE unless it is a rep-prefixed MOVS or + * STOS; otherwise also stores the element size (1, 2, 4 or 8). */ +ClgRepBulkOp clg_rep_bulk_classify(const UChar* bytes, UInt len, + UChar* elem_width); #endif /* CLG_REP_BULK_H */ From 604621d3951f5af050a0901210d208ba40460d33 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 5 Nov 2026 12:08:41 +0100 Subject: [PATCH 4/8] fixup! fix(callgrind): model REP copies with calibrated bulk costs Call CLG_(log_rep_bulk) directly instead of through cachesim_if. It is a cost-model hook rather than a cache simulator entry, so the function pointer, its debug name and the registration block are not needed; --rep-bulk-cost alone gates the call. --- callgrind/global.h | 12 ++++-------- callgrind/main.c | 6 +++--- callgrind/sim.c | 12 +++--------- 3 files changed, 10 insertions(+), 20 deletions(-) diff --git a/callgrind/global.h b/callgrind/global.h index 1727e50c0b..b8c8600f18 100644 --- a/callgrind/global.h +++ b/callgrind/global.h @@ -701,18 +701,10 @@ struct cachesim_if void (*log_0I1Dr)(InstrInfo*, Addr, Word) VG_REGPARM(3); void (*log_0I1Dw)(InstrInfo*, Addr, Word) VG_REGPARM(3); - /* REP bulk cost model: called once per executed element of a supported - * rep-prefixed bulk copy. Args: inode, repeat count after the decrement - * of this element, source address (dst if none), destination address, - * guest DF flag. Zero when the model is off. Uses the ordinary amd64 - * calling convention (regparms=0), not a register-parm ABI. */ - void (*log_rep_bulk)(InstrInfo*, ULong, Addr, Addr, Word); - // function names of helpers (for debugging generated code) const HChar *log_1I0D_name, *log_2I0D_name, *log_3I0D_name; const HChar *log_1I1Dr_name, *log_1I1Dw_name; const HChar *log_0I1Dr_name, *log_0I1Dw_name; - const HChar *log_rep_bulk_name; }; // Event groups @@ -749,6 +741,10 @@ void CLG_(print_debug_usage)(void); /* from sim.c */ void CLG_(init_eventsets)(void); +/* REP bulk cost model: called once per executed element of a classified + * REP instruction. Args: inode, repeat count after the decrement of this + * element, source address (dst if none), destination address, guest DF. */ +void CLG_(log_rep_bulk)(InstrInfo*, ULong, Addr, Addr, Word); /* from main.c */ Bool CLG_(get_debug_info)(Addr, const HChar **dirname, diff --git a/callgrind/main.c b/callgrind/main.c index 1b4ebf0336..add0430999 100644 --- a/callgrind/main.c +++ b/callgrind/main.c @@ -616,7 +616,7 @@ static void flushEvents ( ClgState* clgs ) * *after* VEX's decrement for this element (VEX decrements before the * access), so the helper adds 1 back. */ #if defined(VGA_amd64) - if (CLG_(cachesim).log_rep_bulk) { + if (CLG_(clo).rep_bulk_cost) { InstrInfo* rii = 0; IRExpr* src = 0; IRExpr* dst = 0; @@ -652,8 +652,8 @@ static void flushEvents ( ClgState* clgs ) dflag_tmp, IRExpr_Get(CLG_REP_DFLAG_OFF, Ity_I64))); di = unsafeIRDirty_0_N( - 0, CLG_(cachesim).log_rep_bulk_name, - VG_(fnptr_to_fnentry)(CLG_(cachesim).log_rep_bulk), + 0, "log_rep_bulk", + VG_(fnptr_to_fnentry)(&CLG_(log_rep_bulk)), mkIRExprVec_5( mkIRExpr_HWord((HWord)rii), IRExpr_RdTmp(count_tmp), src ? src : dst, dst, diff --git a/callgrind/sim.c b/callgrind/sim.c index d7cb2d5add..068b7c8bae 100644 --- a/callgrind/sim.c +++ b/callgrind/sim.c @@ -1300,11 +1300,10 @@ static unsigned rep_align_class(Addr src, Addr dst) return CLG_REP_ALIGN_OTHER; } -/* Called once per executed element of a supported rep-prefixed bulk copy; - * `rem_after` is the repeat count after VEX decremented it. The curve delta +/* `rem_after` is the repeat count after VEX decremented it. The curve delta * is charged to both Ct and Cl, so the consumer's sqrt(Ct*Cl) recovers it. */ -static void log_rep_bulk(InstrInfo* ii, ULong rem_after, Addr src, Addr dst, - Word dflag) +void CLG_(log_rep_bulk)(InstrInfo* ii, ULong rem_after, Addr src, Addr dst, + Word dflag) { exec_state* es = &CLG_(current_state); unsigned op = ii->rep_bulk_op; @@ -1451,11 +1450,6 @@ static void cachesim_post_clo_init(void) CLG_(cachesim).log_0I1Dr_name = "log_0I1Dr"; CLG_(cachesim).log_0I1Dw_name = "log_0I1Dw"; - if (CLG_(clo).rep_bulk_cost) { - CLG_(cachesim).log_rep_bulk = log_rep_bulk; - CLG_(cachesim).log_rep_bulk_name = "log_rep_bulk"; - } - if (clo_collect_cacheuse) { /* Output warning for not supported option combinations */ From f44047742d2c8132f24ec666ab1d49ab068179cf Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 5 Nov 2026 12:09:02 +0100 Subject: [PATCH 5/8] fixup! fix(callgrind): model REP copies with calibrated bulk costs Classify REP instructions before the cycle estimate and skip clg_cycle_cost for them, instead of decoding them and zeroing ct_cost/cl_cost afterwards. --- callgrind/main.c | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/callgrind/main.c b/callgrind/main.c index add0430999..0b05082743 100644 --- a/callgrind/main.c +++ b/callgrind/main.c @@ -1117,29 +1117,22 @@ IRSB* CLG_(instrument)( VgCallbackClosure* closure, // also use it. curr_inode = next_InstrInfo (&clgs, isize); + // Classify rep movs/stos for the REP bulk cost model. Their bulk + // curve replaces the per-instruction cycle estimate below. + if (CLG_(clo).rep_bulk_cost && !clgs.seen_before) + curr_inode->rep_bulk_op = clg_rep_bulk_classify( + (const UChar*)cia, isize, &curr_inode->rep_elem_width); + // Estimate this instruction's cycle cost by decoding the real // guest bytes. Only on first translation; values persist in the BB. - if (CLG_(clo).cycle_estimation && !clgs.seen_before) { + if (CLG_(clo).cycle_estimation && !clgs.seen_before + && !curr_inode->rep_bulk_op) { unsigned ct, cl; unsigned len = st->Ist.IMark.len ? st->Ist.IMark.len : isize; clg_cycle_cost((const unsigned char*)cia, len, &ct, &cl); curr_inode->ct_cost = ct; curr_inode->cl_cost = cl; } - - // Classify supported rep-prefixed bulk copies (rep movs/rep stos) for - // the REP bulk cost model. The decision is static per instruction. - if (CLG_(clo).rep_bulk_cost && !clgs.seen_before) { - curr_inode->rep_bulk_op = clg_rep_bulk_classify( - (const UChar*)cia, isize, &curr_inode->rep_elem_width); - if (curr_inode->rep_bulk_op != CLG_REP_OP_NONE) { - /* The calibrated bulk curve replaces this instruction's - * static per-element cycle estimate completely: drop it from - * both the dump-time charge and the inclusive prefix sums. */ - curr_inode->ct_cost = 0; - curr_inode->cl_cost = 0; - } - } addEvent_Ir( &clgs, curr_inode ); break; } From fc6955f2f0e62e28125adc21799421e5fda99de1 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 5 Nov 2026 12:10:33 +0100 Subject: [PATCH 6/8] fixup! fix(callgrind): model REP copies with calibrated bulk costs Emit the REP bulk helper at the instruction's IMark and read RCX, RSI, RDI and DFLAG from the guest state there, instead of scanning the flushed events for the REP inode and its addresses. Before VEX's decrement RCX is the remaining count, so the helper no longer adds 1 back and returns early when it is 0. The 67 prefix is no longer classified: its counter and pointers are ECX/ESI/EDI, and at the IMark the upper halves of RCX/RSI/RDI are not cleared yet. Such instructions (x32 only) keep the per-element model. rep_bulk test and a perl workload with large memcpy produce identical per-part counters. --- callgrind/global.h | 6 ++-- callgrind/main.c | 86 +++++++++++++++----------------------------- callgrind/rep_bulk.c | 4 +-- callgrind/sim.c | 17 +++++---- 4 files changed, 44 insertions(+), 69 deletions(-) diff --git a/callgrind/global.h b/callgrind/global.h index b8c8600f18..081283975d 100644 --- a/callgrind/global.h +++ b/callgrind/global.h @@ -741,9 +741,9 @@ void CLG_(print_debug_usage)(void); /* from sim.c */ void CLG_(init_eventsets)(void); -/* REP bulk cost model: called once per executed element of a classified - * REP instruction. Args: inode, repeat count after the decrement of this - * element, source address (dst if none), destination address, guest DF. */ +/* REP bulk cost model: called at the IMark of each execution of a classified + * REP instruction. Args: inode, repeat count before VEX's decrement, source + * address (dst for STOS), destination address, guest DF. */ void CLG_(log_rep_bulk)(InstrInfo*, ULong, Addr, Addr, Word); /* from main.c */ diff --git a/callgrind/main.c b/callgrind/main.c index 0b05082743..574487fb04 100644 --- a/callgrind/main.c +++ b/callgrind/main.c @@ -43,13 +43,11 @@ #include "rep_bulk.h" #if defined(VGA_amd64) -/* REP bulk cost model: read the repeat counter and DF from the guest state. - * VEX translates rep movs/stos on amd64 as one element per BB execution with - * an unconditional jump back to the instruction, and uses RCX (or ECX with a - * 67 prefix) as the repeat counter, decremented before the element access. */ +/* REP bulk cost model: VEX translates rep movs/stos on amd64 as one element + * per BB execution that jumps back to the instruction, and decrements RCX + * after its IMark. */ #include "libvex_guest_amd64.h" -#define CLG_REP_COUNT_OFF offsetof(VexGuestAMD64State, guest_RCX) -#define CLG_REP_DFLAG_OFF offsetof(VexGuestAMD64State, guest_DFLAG) +#define CLG_GUEST_OFF(reg) offsetof(VexGuestAMD64State, guest_##reg) #endif /*------------------------------------------------------------*/ @@ -610,60 +608,32 @@ static void flushEvents ( ClgState* clgs ) addStmtToIRSB( clgs->sbOut, IRStmt_Dirty(di) ); } - /* REP bulk cost model: charge one calibrated bulk delta per executed - * element. Emitted after the normal memory helpers; the cache simulation - * is unchanged. `Get(RCX)` is the repeat count - * *after* VEX's decrement for this element (VEX decrements before the - * access), so the helper adds 1 back. */ -#if defined(VGA_amd64) - if (CLG_(clo).rep_bulk_cost) { - InstrInfo* rii = 0; - IRExpr* src = 0; - IRExpr* dst = 0; - Int j; - - for (j = clgs->events_used - 1; j >= 0; j--) { - Event* e = &clgs->events[j]; - if (e->tag != Ev_Dr && e->tag != Ev_Dw && e->tag != Ev_Dm) - continue; - if (!rii) - rii = e->inode; - if (e->inode != rii) - continue; - if (e->tag == Ev_Dr) { - if (!src) - src = e->Ev.Dr.ea; - } else { - if (!dst) - dst = get_Event_dea(e); - } - } + clgs->events_used = 0; +} - if (rii && rii->rep_bulk_op && dst) { - /* The two addresses only classify alignment once per operation; - * STOS and merged read-modify events have no separate source. - * A 67-prefixed REP needs no mask: VEX writes the decremented ECX - * zero-extended into RCX before the element's access. */ - IRTemp count_tmp = newIRTemp(clgs->sbOut->tyenv, Ity_I64); - IRTemp dflag_tmp = newIRTemp(clgs->sbOut->tyenv, Ity_I64); - addStmtToIRSB(clgs->sbOut, IRStmt_WrTmp( - count_tmp, IRExpr_Get(CLG_REP_COUNT_OFF, Ity_I64))); - addStmtToIRSB(clgs->sbOut, IRStmt_WrTmp( - dflag_tmp, IRExpr_Get(CLG_REP_DFLAG_OFF, Ity_I64))); - - di = unsafeIRDirty_0_N( - 0, "log_rep_bulk", - VG_(fnptr_to_fnentry)(&CLG_(log_rep_bulk)), - mkIRExprVec_5( mkIRExpr_HWord((HWord)rii), - IRExpr_RdTmp(count_tmp), - src ? src : dst, dst, - IRExpr_RdTmp(dflag_tmp) ) ); - addStmtToIRSB( clgs->sbOut, IRStmt_Dirty(di) ); - } +/* REP bulk cost model: charge the element about to execute. At the IMark, + * before VEX's decrement, RCX counts the elements still to go. STOS has no + * source and passes RDI for both pointers. */ +static void addRepBulkCall ( ClgState* clgs, InstrInfo* inode ) +{ +#if defined(VGA_amd64) + Int offs[4] = { CLG_GUEST_OFF(RCX), CLG_GUEST_OFF(RSI), + CLG_GUEST_OFF(RDI), CLG_GUEST_OFF(DFLAG) }; + IRExpr* args[4]; + Int i; + + if (inode->rep_bulk_op == CLG_REP_OP_STOS) + offs[1] = CLG_GUEST_OFF(RDI); + for (i = 0; i < 4; i++) { + IRTemp t = newIRTemp(clgs->sbOut->tyenv, Ity_I64); + addStmtToIRSB(clgs->sbOut, IRStmt_WrTmp(t, IRExpr_Get(offs[i], Ity_I64))); + args[i] = IRExpr_RdTmp(t); } + addStmtToIRSB(clgs->sbOut, IRStmt_Dirty(unsafeIRDirty_0_N( + 0, "log_rep_bulk", VG_(fnptr_to_fnentry)(&CLG_(log_rep_bulk)), + mkIRExprVec_5(mkIRExpr_HWord((HWord)inode), + args[0], args[1], args[2], args[3])))); #endif - - clgs->events_used = 0; } static void addEvent_Ir ( ClgState* clgs, InstrInfo* inode ) @@ -1122,6 +1092,8 @@ IRSB* CLG_(instrument)( VgCallbackClosure* closure, if (CLG_(clo).rep_bulk_cost && !clgs.seen_before) curr_inode->rep_bulk_op = clg_rep_bulk_classify( (const UChar*)cia, isize, &curr_inode->rep_elem_width); + if (curr_inode->rep_bulk_op) + addRepBulkCall(&clgs, curr_inode); // Estimate this instruction's cycle cost by decoding the real // guest bytes. Only on first translation; values persist in the BB. diff --git a/callgrind/rep_bulk.c b/callgrind/rep_bulk.c index d9f1b5cf61..be1b760a2f 100644 --- a/callgrind/rep_bulk.c +++ b/callgrind/rep_bulk.c @@ -90,7 +90,8 @@ ULong clg_rep_bulk_marginal(unsigned op, unsigned elem_width, } /* Prefixes, then A4/A5 (MOVS) or AA/AB (STOS) as the last byte; REX must - * directly precede the opcode. */ + * directly precede the opcode. The 67 prefix (ECX/ESI/EDI) is not modeled: + * the helper reads the full 64-bit registers. */ ClgRepBulkOp clg_rep_bulk_classify(const UChar* bytes, UInt len, UChar* elem_width) { @@ -102,7 +103,6 @@ ClgRepBulkOp clg_rep_bulk_classify(const UChar* bytes, UInt len, switch (bytes[i]) { case 0xF3: rep = True; break; case 0x66: opsz16 = True; break; - case 0x67: break; default: if ((bytes[i] & 0xF0) != 0x40 || i + 2 != len) return CLG_REP_OP_NONE; diff --git a/callgrind/sim.c b/callgrind/sim.c index 068b7c8bae..ea3b707c6d 100644 --- a/callgrind/sim.c +++ b/callgrind/sim.c @@ -1300,35 +1300,38 @@ static unsigned rep_align_class(Addr src, Addr dst) return CLG_REP_ALIGN_OTHER; } -/* `rem_after` is the repeat count after VEX decremented it. The curve delta - * is charged to both Ct and Cl, so the consumer's sqrt(Ct*Cl) recovers it. */ -void CLG_(log_rep_bulk)(InstrInfo* ii, ULong rem_after, Addr src, Addr dst, +/* `rem` is the repeat count before VEX decrements it, so 0 means the + * instruction finishes without another element. The curve delta is charged + * to both Ct and Cl, so the consumer's sqrt(Ct*Cl) recovers it. */ +void CLG_(log_rep_bulk)(InstrInfo* ii, ULong rem, Addr src, Addr dst, Word dflag) { exec_state* es = &CLG_(current_state); unsigned op = ii->rep_bulk_op; unsigned width = ii->rep_elem_width; - ULong rem_before = rem_after + 1; unsigned dir = ((Long)dflag < 0) ? CLG_REP_DIR_BACKWARD : CLG_REP_DIR_FORWARD; ULong delta; ULong * cost_C, *global_C; + if (rem == 0) + return; + /* The alignment class is latched from the operation's first element, so * address changes between elements cannot switch curves. The next * element continues iff it reports the count and destination expected * after the previous one; the instruction site cannot be compared since * VEX translates the first element as part of the preceding block. */ - if (es->rep_next_rem != rem_before || es->rep_next_dst != dst) + if (es->rep_next_rem != rem || es->rep_next_dst != dst) es->rep_align = rep_align_class(src, dst); - es->rep_next_rem = rem_before - 1; + es->rep_next_rem = rem - 1; es->rep_next_dst = dir == CLG_REP_DIR_BACKWARD ? dst - width : dst + width; if (!es->collect) return; delta = clg_rep_bulk_marginal(op, width, dir, es->rep_align, - rem_before * (ULong)width, (ULong)width); + rem * (ULong)width, (ULong)width); if (es->nonskipped) cost_C = es->nonskipped->skipped + fullOffset(EG_CYCLES); From 2997dbbb03441bcbca5d4905696528ac2b04f5c6 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 5 Nov 2026 12:11:15 +0100 Subject: [PATCH 7/8] fixup! fix(callgrind): model REP copies with calibrated bulk costs Trim the rep_bulk test: drop the self-copy part, which takes the same path as the hot forward copy, and compact rep_bulk_check.pl (hash-slice parsing, one check helper, shared cold-miss range check). The on/off checks are unchanged apart from the renumbered cold part. --- callgrind/tests/rep_bulk.c | 9 +-- callgrind/tests/rep_bulk.post.exp | 1 - callgrind/tests/rep_bulk_check.pl | 97 ++++++++----------------------- 3 files changed, 24 insertions(+), 83 deletions(-) diff --git a/callgrind/tests/rep_bulk.c b/callgrind/tests/rep_bulk.c index 5a23462a85..984fe3fc6b 100644 --- a/callgrind/tests/rep_bulk.c +++ b/callgrind/tests/rep_bulk.c @@ -15,7 +15,6 @@ #define COPY_SZ 10240u /* 160 x 64B lines */ #define BACK_SZ 4096u #define STOS_SZ 4096u -#define SELF_SZ 4096u #define FLUSH_SZ (256u * 1024u) static unsigned char srcbuf[COPY_SZ + 128] __attribute__((aligned(64))); @@ -120,13 +119,7 @@ int main(void) if (dstbuf[i] != 0x5a) return 6; - /* part 5: self copy (source and destination are the same buffer) */ - measure_copy(dstbuf, dstbuf, SELF_SZ); - for (i = 0; i < SELF_SZ; i++) - if (dstbuf[i] != 0x5a) - return 7; - - /* part 6: cold 10 KiB forward copy after flushing D1 */ + /* part 5: cold 10 KiB forward copy after flushing D1 */ flush_d1(); measure_copy(dstbuf, srcbuf, COPY_SZ); if (!verify_copy()) diff --git a/callgrind/tests/rep_bulk.post.exp b/callgrind/tests/rep_bulk.post.exp index 17913a8e18..e0045a8279 100644 --- a/callgrind/tests/rep_bulk.post.exp +++ b/callgrind/tests/rep_bulk.post.exp @@ -5,6 +5,5 @@ instruction_refs_kept: ok bulk_cycles: ok backward: ok stos: ok -self: ok cold_miss_refs: ok cache_independent_cycles: ok diff --git a/callgrind/tests/rep_bulk_check.pl b/callgrind/tests/rep_bulk_check.pl index 29edbc83ca..68797dd067 100755 --- a/callgrind/tests/rep_bulk_check.pl +++ b/callgrind/tests/rep_bulk_check.pl @@ -1,122 +1,71 @@ #!/usr/bin/perl # Check the callgrind output of the rep_bulk test. # Usage: rep_bulk_check.pl on|off + use strict; use warnings; my ($file, $mode) = @ARGV; -die "usage: rep_bulk_check.pl on|off\n" - unless defined $file && defined $mode && ($mode eq 'on' || $mode eq 'off'); - open(my $fh, '<', $file) or die "cannot open $file: $!\n"; -my @events; -my @parts; -while (my $line = <$fh>) { - chomp $line; - if ($line =~ /^events:\s+(.*)$/) { - @events = split(' ', $1) unless @events; - } elsif ($line =~ /^summary:\s+(.*)$/) { - my @v = split(' ', $1); +my (@events, @parts); +while (<$fh>) { + @events = split(' ', $1) if !@events && /^events:\s+(.*)$/; + if (/^summary:\s+(.*)$/) { my %p; - for my $i (0 .. $#events) { - $p{$events[$i]} = $v[$i]; - } + @p{@events} = split(' ', $1); push @parts, \%p; } } close($fh); +die "expected at least 6 dump parts, got " . @parts . "\n" if @parts < 6; -my @fail; -sub check { - my ($name, $ok) = @_; - if ($ok) { - print "$name: ok\n"; - } else { - print "$name: MISMATCH\n"; - push @fail, $name; - } -} sub val { my ($part, $name) = @_; - die "part $part has no event $name\n" unless defined $parts[$part]{$name}; - return $parts[$part]{$name} + 0; + return $parts[$part]{$name} // die "part $part has no event $name\n"; } - -my $expected_events = 'Ir Dr Dw I1mr D1mr D1mw ILmr DLmr DLmw Ct Cl'; -die "expected at least 7 dump parts, got ".scalar(@parts)."\n" if @parts < 7; - -check('events', join(' ', @events) eq $expected_events); - # Deltas of a part against part 2 (zero-length rep: pure wrapper cost). sub delta { my ($part, $name) = @_; return val($part, $name) - val(2, $name); } +# Cold 10 KiB copy: one miss per 64B line (160), well below one per element. +sub cold_misses { return !grep { delta(5, $_) < 160 || delta(5, $_) >= 1024 } @_; } +my $fail = 0; +sub check { + my ($name, $ok) = @_; + print "$name: ", ($ok ? "ok" : "MISMATCH"), "\n"; + $fail = 1 unless $ok; +} + +check('events', "@events" eq 'Ir Dr Dw I1mr D1mr D1mw ILmr DLmr DLmw Ct Cl'); if ($mode eq 'on') { check('one_byte_bulk_cost', val(0, 'Ct') > val(2, 'Ct')); - check('hot_lines_suppressed', - delta(1, 'Dr') == 0 && delta(1, 'Dw') == 0 - && delta(1, 'D1mr') == 0 && delta(1, 'D1mw') == 0); - + !grep { delta(1, $_) != 0 } qw(Dr Dw D1mr D1mw)); # Instruction-side counters are never modeled: one element executes one # more IMark than the zero-length dispatch (1 vs 2 dispatches). check('instruction_refs_kept', delta(0, 'Ir') == 1 && delta(1, 'Ir') == 10240); - # The aligned curve prices 10 KiB at about 19K centicycles, the # misaligned one at about 38K: the class must come from the first element. my $ct = delta(1, 'Ct'); check('bulk_cycles', $ct > 0 && $ct < 25000 && delta(1, 'Cl') > 0); - # A 4 KiB backward copy costs more than the 10 KiB forward one only if # every element keeps the backward curve. check('backward', delta(3, 'Ct') > delta(1, 'Ct') && delta(3, 'Dr') == 0 && delta(3, 'Dw') == 0); - check('stos', val(4, 'Ct') > val(2, 'Ct') && delta(4, 'Dr') == 0 && delta(4, 'Dw') == 0); - - check('self', - val(5, 'Ct') > val(2, 'Ct') - && delta(5, 'Dr') == 0 && delta(5, 'Dw') == 0); - - my $dr = delta(6, 'Dr'); - my $dw = delta(6, 'Dw'); - my $d1r = delta(6, 'D1mr'); - my $d1w = delta(6, 'D1mw'); - check('cold_miss_refs', - $dr >= 160 && $dr < 1024 && - $dw >= 160 && $dw < 1024 && - $d1r >= 160 && $d1r < 1024 && - $d1w >= 160 && $d1w < 1024); - + check('cold_miss_refs', cold_misses(qw(Dr Dw D1mr D1mw))); check('cache_independent_cycles', - val(1, 'Ct') == val(6, 'Ct') && val(1, 'Cl') == val(6, 'Cl')); + val(1, 'Ct') == val(5, 'Ct') && val(1, 'Cl') == val(5, 'Cl')); } else { check('element_refs', delta(0, 'Dr') == 1 && delta(0, 'Dw') == 1); - check('legacy_hot_refs', delta(1, 'Dr') == 10240 && delta(1, 'Dw') == 10240); - check('legacy_cycles', delta(1, 'Ct') > 500000); - check('legacy_zero_dispatch', delta(0, 'Ct') >= 100); - check('legacy_backward_refs', delta(3, 'Dr') == 4096 && delta(3, 'Dw') == 4096); - check('legacy_stos_refs', delta(4, 'Dr') == 0 && delta(4, 'Dw') == 4096); - - - my $dr = delta(6, 'Dr'); - my $d1r = delta(6, 'D1mr'); - my $d1w = delta(6, 'D1mw'); - check('legacy_cold_refs', - $dr >= 10240 && $d1r >= 160 && $d1r < 1024 && - $d1w >= 160 && $d1w < 1024); -} - -if (@fail) { - print "FAILED: ", join(', ', @fail), "\n"; - exit 1; + check('legacy_cold_refs', delta(5, 'Dr') >= 10240 && cold_misses(qw(D1mr D1mw))); } -exit 0; +exit $fail; From 9db6684ac99940852985a4b989d030a9d4f721e9 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 5 Nov 2026 16:56:58 +0100 Subject: [PATCH 8/8] fix(callgrind): report only LL misses inside modeled REP copies The bulk curve covers L1-resident cost. Charging every L1 miss of a REP copy on top priced lines streamed from LL like isolated misses, which overstated large copies by an order of magnitude. Only accesses that miss LL are now reported for modeled REP instructions. --- callgrind/docs/cl-manual.xml | 8 +++++--- callgrind/sim.c | 12 +++++++----- callgrind/tests/rep_bulk.post.exp | 2 +- callgrind/tests/rep_bulk_check.pl | 4 +++- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/callgrind/docs/cl-manual.xml b/callgrind/docs/cl-manual.xml index 148ee883d6..a4e8bc3046 100644 --- a/callgrind/docs/cl-manual.xml +++ b/callgrind/docs/cl-manual.xml @@ -1007,9 +1007,11 @@ Also see . enabled, and not with , or non-amd64 guests. For a modeled instruction, "Ct" and "Cl" both carry the curve - cost, and only data accesses that miss in L1 are reported as "Dr"/"Dw" - (so Dr equals D1mr and Dw equals D1mw); instruction-side counters and - the cache simulation are unchanged. + cost, and only data accesses that miss in LL are reported as "Dr"/"Dw" + (so Dr equals D1mr equals DLmr, likewise for writes): an L1 miss that + hits LL streams at near-hit cost in a REP copy. Instruction-side + counters and the cache simulation are unchanged. + reproduces the per-element model exactly. The curve was measured on disjoint, cache-resident buffers up to 16 KiB (MOVS) and 32 KiB (STOS); larger copies extend the last measured diff --git a/callgrind/sim.c b/callgrind/sim.c index ea3b707c6d..463438da9d 100644 --- a/callgrind/sim.c +++ b/callgrind/sim.c @@ -1182,8 +1182,10 @@ static void log_1I1Dr(InstrInfo* ii, Addr data_addr, Word data_size) inc_costs(IrRes, cost_Ir, CLG_(current_state).cost + fullOffset(EG_IR) ); - /* REP bulk model: keep data misses, the curve replaces L1-hit refs. */ - if (!ii->rep_bulk_op || DrRes != L1_Hit) + /* REP bulk model: the curve replaces L1-hit refs, and an L1 miss that + * hits LL streams at near-hit cost in a REP copy, so only accesses + * that miss LL are reported. */ + if (!ii->rep_bulk_op || DrRes == MemAccess) inc_costs(DrRes, cost_Dr, CLG_(current_state).cost + fullOffset(EG_DR) ); } @@ -1212,7 +1214,7 @@ static void log_0I1Dr(InstrInfo* ii, Addr data_addr, Word data_size) else cost_Dr = CLG_(cost_base) + ii->cost_offset + ii->eventset->offset[EG_DR]; - if (!ii->rep_bulk_op || DrRes != L1_Hit) + if (!ii->rep_bulk_op || DrRes == MemAccess) inc_costs(DrRes, cost_Dr, CLG_(current_state).cost + fullOffset(EG_DR) ); } @@ -1248,7 +1250,7 @@ static void log_1I1Dw(InstrInfo* ii, Addr data_addr, Word data_size) inc_costs(IrRes, cost_Ir, CLG_(current_state).cost + fullOffset(EG_IR) ); - if (!ii->rep_bulk_op || DwRes != L1_Hit) + if (!ii->rep_bulk_op || DwRes == MemAccess) inc_costs(DwRes, cost_Dw, CLG_(current_state).cost + fullOffset(EG_DW) ); } @@ -1274,7 +1276,7 @@ static void log_0I1Dw(InstrInfo* ii, Addr data_addr, Word data_size) else cost_Dw = CLG_(cost_base) + ii->cost_offset + ii->eventset->offset[EG_DW]; - if (!ii->rep_bulk_op || DwRes != L1_Hit) + if (!ii->rep_bulk_op || DwRes == MemAccess) inc_costs(DwRes, cost_Dw, CLG_(current_state).cost + fullOffset(EG_DW) ); } diff --git a/callgrind/tests/rep_bulk.post.exp b/callgrind/tests/rep_bulk.post.exp index e0045a8279..884714fb0d 100644 --- a/callgrind/tests/rep_bulk.post.exp +++ b/callgrind/tests/rep_bulk.post.exp @@ -5,5 +5,5 @@ instruction_refs_kept: ok bulk_cycles: ok backward: ok stos: ok -cold_miss_refs: ok +ll_resident_refs_suppressed: ok cache_independent_cycles: ok diff --git a/callgrind/tests/rep_bulk_check.pl b/callgrind/tests/rep_bulk_check.pl index 68797dd067..d7f74b842c 100755 --- a/callgrind/tests/rep_bulk_check.pl +++ b/callgrind/tests/rep_bulk_check.pl @@ -56,7 +56,9 @@ sub check { check('stos', val(4, 'Ct') > val(2, 'Ct') && delta(4, 'Dr') == 0 && delta(4, 'Dw') == 0); - check('cold_miss_refs', cold_misses(qw(Dr Dw D1mr D1mw))); + # L1-cold but LL-resident lines stream at near-hit cost: no refs. + check('ll_resident_refs_suppressed', + !grep { delta(5, $_) != 0 } qw(Dr Dw D1mr D1mw)); check('cache_independent_cycles', val(1, 'Ct') == val(5, 'Ct') && val(1, 'Cl') == val(5, 'Cl')); } else {