diff --git a/CODSPEED-CHANGELOG.md b/CODSPEED-CHANGELOG.md index 61808930c4..63f3fdde9d 100644 --- a/CODSPEED-CHANGELOG.md +++ b/CODSPEED-CHANGELOG.md @@ -23,6 +23,13 @@ matching Linux on AArch64. ## Features +### REP bulk cost model for `rep movs` / `rep stos` + +Callgrind now prices amd64 `rep movs` and `rep stos` from a calibrated +hot-copy curve when cache simulation and `--cycle-estimation=yes` are both +active: `Ct`/`Cl` carry the curve cost, and only data L1 misses are reported +as `Dr`/`Dw`. `--rep-bulk-cost=no` restores the per-element model. + ### Fixed Linux startup stack layout Reduce simulated cache-miss variation from argument and environment sizes diff --git a/callgrind/Makefile.am b/callgrind/Makefile.am index de92e16b89..0b90003128 100644 --- a/callgrind/Makefile.am +++ b/callgrind/Makefile.am @@ -22,11 +22,14 @@ noinst_HEADERS = \ cycledecode_capstone.h \ sigkey.h \ events.h \ - global.h + global.h \ + rep_bulk.h # Generated cost tables (#included by cycledecode.c under CLG_WITH_CAPSTONE; -# the arch is selected at compile time). -EXTRA_DIST += x86_caps_lut.inc arm64_caps_lut.inc +# the arch is selected at compile time) and the calibrated REP bulk curve +# table included by rep_bulk.c (generated by the instruction cost calibration +# pipeline). +EXTRA_DIST += x86_caps_lut.inc arm64_caps_lut.inc rep_bulk_table.inc #---------------------------------------------------------------------------- # callgrind- @@ -52,6 +55,7 @@ CALLGRIND_SOURCES_COMMON = \ fn.c \ jumps.c \ main.c \ + rep_bulk.c \ sim.c \ subprocess.c \ threads.c diff --git a/callgrind/clo.c b/callgrind/clo.c index 911051a27d..8549049a2c 100644 --- a/callgrind/clo.c +++ b/callgrind/clo.c @@ -568,6 +568,7 @@ Bool CLG_(process_cmd_line_option)(const HChar* arg) else if VG_BOOL_CLO(arg, "--branch-sim", CLG_(clo).simulate_branch) {} else if VG_BOOL_CLO(arg, "--cycle-estimation", CLG_(clo).cycle_estimation) {} + else if VG_BOOL_CLO(arg, "--rep-bulk-cost", CLG_(clo).rep_bulk_cost) {} else { Bool isCachesimOption = (*CLG_(cachesim).parse_opt)(arg); @@ -617,6 +618,8 @@ void CLG_(print_usage)(void) " --collect-jumps=no|yes Collect jumps? [no]\n" " --collect-bus=no|yes Collect global bus events? [no]\n" " --cycle-estimation=no|yes Estimate per-instruction cycles (Ct/Cl)? [no]\n" +" --rep-bulk-cost=no|yes Calibrated rep movs/stos cost, needs cache\n" +" sim and --cycle-estimation=yes [yes]\n" #if CLG_EXPERIMENTAL " --collect-alloc=no|yes Collect memory allocation info? [no]\n" #endif @@ -704,6 +707,7 @@ void CLG_(set_clo_defaults)(void) CLG_(clo).simulate_cache = False; CLG_(clo).simulate_branch = False; CLG_(clo).cycle_estimation = False; + CLG_(clo).rep_bulk_cost = True; /* Call graph */ CLG_(clo).pop_on_jump = False; diff --git a/callgrind/docs/cl-manual.xml b/callgrind/docs/cl-manual.xml index 8b4e50d853..a4e8bc3046 100644 --- a/callgrind/docs/cl-manual.xml +++ b/callgrind/docs/cl-manual.xml @@ -995,6 +995,31 @@ Also see . + + + + + + Model amd64 rep movs and + rep stos with a calibrated bulk cost + curve instead of the per-element model. Active only when cache + simulation and are both + enabled, and not with , + or non-amd64 guests. + For a modeled instruction, "Ct" and "Cl" both carry the curve + cost, and only data accesses that miss in LL are reported as "Dr"/"Dw" + (so Dr equals D1mr equals DLmr, likewise for writes): an L1 miss that + hits LL streams at near-hit cost in a REP copy. Instruction-side + counters and the cache simulation are unchanged. + + reproduces the per-element model exactly. + The curve was measured on disjoint, cache-resident buffers up to + 16 KiB (MOVS) and 32 KiB (STOS); larger copies extend the last measured + slope. Overlapping or cache-miss-heavy copies are outside the + calibrated domain. + + + diff --git a/callgrind/global.h b/callgrind/global.h index 97a22bf252..081283975d 100644 --- a/callgrind/global.h +++ b/callgrind/global.h @@ -128,6 +128,7 @@ struct _CommandLineOptions { Bool simulate_cache; /* Call into cache simulator ? */ Bool simulate_branch; /* Call into branch prediction simulator ? */ Bool cycle_estimation; /* Estimate per-instruction cycles (Ct/Cl) ? */ + Bool rep_bulk_cost; /* Replace per-element REP cost by bulk curve ? */ /* Call graph generation */ Bool pop_on_jump; /* Handle a jump between functions as ret+call */ @@ -279,6 +280,11 @@ struct _InstrInfo { UInt ct_incl; UInt cl_cost; UInt cl_incl; + /* REP bulk cost model classification (see rep_bulk.h). Only filled on + * first translation and only when the model is active; rep_bulk_op == 0 + * means the instruction is not a supported rep-prefixed bulk copy. */ + UChar rep_bulk_op; + UChar rep_elem_width; }; @@ -528,6 +534,14 @@ struct _exec_state { BBCC* nonskipped; Int call_stack_bottom; /* Index into fn_stack */ + + /* REP bulk cost model: the alignment class is latched once per REP + * operation from its first element, and the expected remaining count and + * destination detect the next operation. Saved/restored on thread and + * signal switches so an interrupted REP keeps its class. */ + Addr rep_next_dst; /* expected destination of the next element */ + ULong rep_next_rem; /* expected remaining count after last element */ + UChar rep_align; /* CLG_REP_ALIGN_* latched at operation start */ }; /* Global state structures */ @@ -727,6 +741,10 @@ void CLG_(print_debug_usage)(void); /* from sim.c */ void CLG_(init_eventsets)(void); +/* REP bulk cost model: called at the IMark of each execution of a classified + * REP instruction. Args: inode, repeat count before VEX's decrement, source + * address (dst for STOS), destination address, guest DF. */ +void CLG_(log_rep_bulk)(InstrInfo*, ULong, Addr, Addr, Word); /* from main.c */ Bool CLG_(get_debug_info)(Addr, const HChar **dirname, diff --git a/callgrind/main.c b/callgrind/main.c index 69af08255f..574487fb04 100644 --- a/callgrind/main.c +++ b/callgrind/main.c @@ -40,6 +40,15 @@ #include "cg_branchpred.c" #include "cycledecode.h" +#include "rep_bulk.h" + +#if defined(VGA_amd64) +/* REP bulk cost model: VEX translates rep movs/stos on amd64 as one element + * per BB execution that jumps back to the instruction, and decrements RCX + * after its IMark. */ +#include "libvex_guest_amd64.h" +#define CLG_GUEST_OFF(reg) offsetof(VexGuestAMD64State, guest_##reg) +#endif /*------------------------------------------------------------*/ /*--- Global variables ---*/ @@ -397,6 +406,11 @@ static void flushEvents ( ClgState* clgs ) // Ir event always is first for a guest instruction CLG_ASSERT(ev->inode->eventset == 0); ev->inode->eventset = CLG_(sets).base; + /* REP bulk model: REP instructions carry per-instruction Ct/Cl + * slots, filled at runtime from the calibrated bulk curve. */ + if (ev->inode->rep_bulk_op) + ev->inode->eventset = + CLG_(add_event_group)(ev->inode->eventset, EG_CYCLES); break; case Ev_Dr: // extend event set by Dr counters @@ -597,6 +611,31 @@ static void flushEvents ( ClgState* clgs ) clgs->events_used = 0; } +/* REP bulk cost model: charge the element about to execute. At the IMark, + * before VEX's decrement, RCX counts the elements still to go. STOS has no + * source and passes RDI for both pointers. */ +static void addRepBulkCall ( ClgState* clgs, InstrInfo* inode ) +{ +#if defined(VGA_amd64) + Int offs[4] = { CLG_GUEST_OFF(RCX), CLG_GUEST_OFF(RSI), + CLG_GUEST_OFF(RDI), CLG_GUEST_OFF(DFLAG) }; + IRExpr* args[4]; + Int i; + + if (inode->rep_bulk_op == CLG_REP_OP_STOS) + offs[1] = CLG_GUEST_OFF(RDI); + for (i = 0; i < 4; i++) { + IRTemp t = newIRTemp(clgs->sbOut->tyenv, Ity_I64); + addStmtToIRSB(clgs->sbOut, IRStmt_WrTmp(t, IRExpr_Get(offs[i], Ity_I64))); + args[i] = IRExpr_RdTmp(t); + } + addStmtToIRSB(clgs->sbOut, IRStmt_Dirty(unsafeIRDirty_0_N( + 0, "log_rep_bulk", VG_(fnptr_to_fnentry)(&CLG_(log_rep_bulk)), + mkIRExprVec_5(mkIRExpr_HWord((HWord)inode), + args[0], args[1], args[2], args[3])))); +#endif +} + static void addEvent_Ir ( ClgState* clgs, InstrInfo* inode ) { Event* evt; @@ -795,6 +834,8 @@ InstrInfo* next_InstrInfo ( ClgState* clgs, UInt instr_size ) ii->ct_incl = 0; ii->cl_cost = 0; ii->cl_incl = 0; + ii->rep_bulk_op = 0; + ii->rep_elem_width = 0; } clgs->ii_index++; @@ -1046,9 +1087,18 @@ IRSB* CLG_(instrument)( VgCallbackClosure* closure, // also use it. curr_inode = next_InstrInfo (&clgs, isize); + // Classify rep movs/stos for the REP bulk cost model. Their bulk + // curve replaces the per-instruction cycle estimate below. + if (CLG_(clo).rep_bulk_cost && !clgs.seen_before) + curr_inode->rep_bulk_op = clg_rep_bulk_classify( + (const UChar*)cia, isize, &curr_inode->rep_elem_width); + if (curr_inode->rep_bulk_op) + addRepBulkCall(&clgs, curr_inode); + // Estimate this instruction's cycle cost by decoding the real // guest bytes. Only on first translation; values persist in the BB. - if (CLG_(clo).cycle_estimation && !clgs.seen_before) { + if (CLG_(clo).cycle_estimation && !clgs.seen_before + && !curr_inode->rep_bulk_op) { unsigned ct, cl; unsigned len = st->Ist.IMark.len ? st->Ist.IMark.len : isize; clg_cycle_cost((const unsigned char*)cia, len, &ct, &cl); diff --git a/callgrind/rep_bulk.c b/callgrind/rep_bulk.c new file mode 100644 index 0000000000..be1b760a2f --- /dev/null +++ b/callgrind/rep_bulk.c @@ -0,0 +1,118 @@ +/*--------------------------------------------------------------------*/ +/*--- REP bulk cost model: calibrated hot-copy curve. ---*/ +/*--- rep_bulk.c ---*/ +/*--------------------------------------------------------------------*/ + +/* CodSpeed extension. See rep_bulk.h. */ + +#include "rep_bulk.h" +#include "pub_tool_libcassert.h" + +/* Generated by the calibration pipeline. Defines + * clg_rep_bulk_provenance[], clg_rep_bulk_pts_[] and + * clg_rep_bulk_curves[] (types from rep_bulk.h). */ +#include "rep_bulk_table.inc" + +#define N_CURVES ((UInt)(sizeof(clg_rep_bulk_curves) / \ + sizeof(clg_rep_bulk_curves[0]))) + +#define N_WIDTHS 4 +#define N_DIRS 2 +#define N_ALIGN 3 + +/* The generated table lists the curves in key order: op, width, direction, + * alignment class. */ +static const ClgRepBulkCurve* curve(unsigned op, unsigned elem_width, + unsigned direction, unsigned align_class) +{ + UInt w = (UInt)__builtin_ctz(elem_width); + return &clg_rep_bulk_curves[(((op - CLG_REP_OP_MOVS) * N_WIDTHS + w) + * N_DIRS + direction) * N_ALIGN + align_class]; +} + +/* Interpolate linearly between the neighbouring sampled byte sizes. Integer + * floor rounding keeps the result monotone, so marginal costs never become + * negative (consecutive element deltas telescope exactly for the same F). */ +static ULong cost_at(const ClgRepBulkCurve* c, ULong bytes) +{ + const ClgRepBulkPoint* p = c->pts; + UInt n = c->npts; + + if (bytes <= p[0].bytes) + return p[0].centicycles; + + /* Last point <= bytes. */ + UInt lo = 0, hi = n - 1; + while (lo < hi) { + UInt mid = lo + (hi - lo + 1) / 2; + if (p[mid].bytes <= bytes) + lo = mid; + else + hi = mid - 1; + } + + if (p[lo].bytes == bytes) + return p[lo].centicycles; + + /* Beyond the last sampled size, extend the last segment's slope. */ + if (lo + 1 == n) + lo--; + + ULong dbytes = (ULong)(p[lo + 1].bytes - p[lo].bytes); + ULong dcost = p[lo + 1].centicycles - p[lo].centicycles; + ULong off = bytes - p[lo].bytes; + return p[lo].centicycles + (off * dcost) / dbytes; +} + +void clg_rep_bulk_init(void) +{ + UInt i; + + (void)clg_rep_bulk_provenance; + tl_assert(N_CURVES == 2 * N_WIDTHS * N_DIRS * N_ALIGN); + for (i = 0; i < N_CURVES; i++) { + const ClgRepBulkCurve* c = &clg_rep_bulk_curves[i]; + tl_assert(c->npts >= 2); + tl_assert(curve(c->key.op, c->key.elem_width, c->key.direction, + c->key.align_class) == c); + } +} + +ULong clg_rep_bulk_marginal(unsigned op, unsigned elem_width, + unsigned direction, unsigned align_class, + ULong remaining_bytes, ULong elem_bytes) +{ + const ClgRepBulkCurve* c = curve(op, elem_width, direction, align_class); + + /* F is monotone, so the difference is never negative and telescopes to + * F(total) over one operation. */ + return cost_at(c, remaining_bytes) - cost_at(c, remaining_bytes - elem_bytes); +} + +/* Prefixes, then A4/A5 (MOVS) or AA/AB (STOS) as the last byte; REX must + * directly precede the opcode. The 67 prefix (ECX/ESI/EDI) is not modeled: + * the helper reads the full 64-bit registers. */ +ClgRepBulkOp clg_rep_bulk_classify(const UChar* bytes, UInt len, + UChar* elem_width) +{ + UChar opc = bytes[len - 1]; + Bool rep = False, opsz16 = False, rex_w = False; + UInt i; + + for (i = 0; i + 1 < len; i++) { + switch (bytes[i]) { + case 0xF3: rep = True; break; + case 0x66: opsz16 = True; break; + default: + if ((bytes[i] & 0xF0) != 0x40 || i + 2 != len) + return CLG_REP_OP_NONE; + rex_w = (bytes[i] & 0x08) != 0; + } + } + + if (!rep || ((opc & ~1) != 0xA4 && (opc & ~1) != 0xAA)) + return CLG_REP_OP_NONE; + /* Even opcodes copy bytes; odd ones use the operand size. */ + *elem_width = (opc & 1) == 0 ? 1 : rex_w ? 8 : opsz16 ? 2 : 4; + return (opc & ~1) == 0xA4 ? CLG_REP_OP_MOVS : CLG_REP_OP_STOS; +} diff --git a/callgrind/rep_bulk.h b/callgrind/rep_bulk.h new file mode 100644 index 0000000000..486804bb32 --- /dev/null +++ b/callgrind/rep_bulk.h @@ -0,0 +1,76 @@ +/*--------------------------------------------------------------------*/ +/*--- REP bulk cost model: calibrated hot-copy curve. ---*/ +/*--- rep_bulk.h ---*/ +/*--------------------------------------------------------------------*/ + +/* CodSpeed extension. + * + * Replaces the per-element cost of rep-prefixed MOVS/STOS with a calibrated + * curve F: the hot-cache cost of one instruction over a given number of + * bytes, in centi-cycles. Each executed element adds + * + * F(remaining_bytes) - F(remaining_bytes - elem_bytes) + * + * which telescopes to F(total_bytes) and stays exact across signal + * interruption, because "remaining" is read from the guest repeat counter. + */ + +#ifndef CLG_REP_BULK_H +#define CLG_REP_BULK_H + +#include "pub_tool_basics.h" + +typedef enum { + CLG_REP_OP_NONE = 0, + CLG_REP_OP_MOVS = 1, /* rep movs{b,w,d,q}: one read + one write per element */ + CLG_REP_OP_STOS = 2 /* rep stos{b,w,d,q}: one write per element */ +} ClgRepBulkOp; + +/* From the guest DF flag, which is constant during an instruction. */ +#define CLG_REP_DIR_FORWARD 0 +#define CLG_REP_DIR_BACKWARD 1 + +/* Alignment class, latched from the first element's pointers (STOS uses + * dst as src): 0 both %64==0, 1 both %64==8, 2 other (conservative). */ +#define CLG_REP_ALIGN_ALIGNED 0 +#define CLG_REP_ALIGN_SAME_UNALIGN 1 +#define CLG_REP_ALIGN_OTHER 2 + +typedef struct { + unsigned char op; /* ClgRepBulkOp */ + unsigned char elem_width; /* 1, 2, 4 or 8 bytes */ + unsigned char direction; /* CLG_REP_DIR_* */ + unsigned char align_class; /* CLG_REP_ALIGN_* */ +} ClgRepBulkKey; + +typedef struct { + UInt bytes; /* cumulative element bytes, strictly ascending */ + ULong centicycles; /* total hot-resident cost, centi-cycles */ +} ClgRepBulkPoint; + +typedef struct { + ClgRepBulkKey key; + const ClgRepBulkPoint* pts; + UInt npts; +} ClgRepBulkCurve; + +/* Assert that the generated table has every key in index order. Call once + * at startup before clg_rep_bulk_marginal. */ +void clg_rep_bulk_init(void); + +/* Marginal bulk cost of the element at `remaining_bytes` (bytes still to + * execute before this element, a nonzero multiple of elem_bytes), in + * centi-cycles. The key must be a classified (op, width) pair with a + * CLG_REP_DIR_* and CLG_REP_ALIGN_* value; beyond the last calibrated + * point the last segment's slope is extended. */ +ULong clg_rep_bulk_marginal(unsigned op, unsigned elem_width, + unsigned direction, unsigned align_class, + ULong remaining_bytes, ULong elem_bytes); + +/* Classify the amd64 guest instruction in [bytes, bytes+len) by its + * encoding. Returns CLG_REP_OP_NONE unless it is a rep-prefixed MOVS or + * STOS; otherwise also stores the element size (1, 2, 4 or 8). */ +ClgRepBulkOp clg_rep_bulk_classify(const UChar* bytes, UInt len, + UChar* elem_width); + +#endif /* CLG_REP_BULK_H */ diff --git a/callgrind/rep_bulk_table.inc b/callgrind/rep_bulk_table.inc new file mode 100644 index 0000000000..fa1bff0b2f --- /dev/null +++ b/callgrind/rep_bulk_table.inc @@ -0,0 +1,880 @@ +/* Generated by lut-gen/x86/repl_bulk/repl_bulk.py -- do not edit. + * + * Schema (from callgrind/rep_bulk.h): + * ClgRepBulkPoint { unsigned bytes; unsigned long long centicycles; } + * ClgRepBulkCurve { ClgRepBulkKey key; const ClgRepBulkPoint *pts; + * unsigned npts; } + * + * F(bytes) is the cumulative hot L1-resident total cost of ONE rep-prefixed + * instruction, in centi-cycles (1 cycle = 100), from dispatch through + * completion. It excludes surrounding code and every L1/LL/RAM miss penalty. + * Per executed element (width w): F(rem*w) - F((rem-1)*w), telescoping. + * Between points the consumer uses integer-linear floor interpolation. + * Beyond the last point cost_at extends the last segment linearly. + * align 0 = both 64B aligned, 1 = both offset 8, 2 = other (conservative + * envelope of odd/split measurements; see provenance). + * + * raw sha256: 17c25bd00601968d162a82a50994c1f293947cb7872cf35566f113a36b362737 + * probe sha256: 2f824fc61dd68a6d229a84091c99f4e445ea832a82f7454e3a80e6211a8903c8 + * Calibration: AMD Ryzen 9 9950X 16-Core Processor, measured fenced completion cost. + * This is NOT the static Zen4 uops.info LUT. + * calibrated max: MOVS 16384 bytes, STOS 32768 bytes + */ +static const char clg_rep_bulk_provenance[] = "repl-bulk-v1 host=AMD Ryzen 9 9950X 16-Core Processor date=2026-11-05T10:44:52.369756+00:00 raw_sha256=17c25bd00601968d162a82a50994c1f293947cb7872cf35566f113a36b362737 probe_sha256=2f824fc61dd68a6d229a84091c99f4e445ea832a82f7454e3a80e6211a8903c8 toolchain=gcc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0 calibrated_max=MOVS:16384,STOS:32768 units=centi-cycles(1cycle=100) regime=hot-L1-resident align2=conservative-envelope >max=carry-linear-tail domain=disjoint-buffers(overlap-not-claimed) cache-miss-penalties-excluded"; + +static const ClgRepBulkPoint clg_rep_bulk_pts_0[] = { + {0,0}, {1,1125}, {127,1125}, {128,3714}, + {192,3714}, {224,3746}, {320,3746}, {384,3820}, + {448,3900}, {640,3900}, {768,4068}, {896,4215}, + {1024,4521}, {1280,4876}, {1536,5228}, {1792,5666}, + {2048,6078}, {2560,6980}, {3072,7784}, {3584,8545}, + {4096,9319}, {5120,10995}, {6144,12645}, {7168,14223}, + {8192,15831}, {10240,19052}, {12288,22270}, {14336,25445}, + {16384,29317}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_1[] = { + {0,0}, {1,1096}, {64,1096}, {65,1099}, + {127,1099}, {128,4198}, {160,4198}, {192,4286}, + {224,4286}, {256,4496}, {320,4527}, {384,4739}, + {448,4913}, {512,5062}, {640,5264}, {768,5458}, + {896,5659}, {1024,5813}, {1280,6261}, {1536,6657}, + {1792,7016}, {2048,7396}, {2560,8042}, {3072,9015}, + {3584,9715}, {4096,10565}, {5120,12180}, {6144,13724}, + {7168,15376}, {8192,17047}, {10240,20188}, {12288,23366}, + {14336,26560}, {16384,30398}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_2[] = { + {0,0}, {1,4916}, {2,4916}, {3,5072}, + {4,5072}, {5,5165}, {8,5165}, {9,5205}, + {10,5205}, {11,5273}, {16,5273}, {17,5320}, + {18,5320}, {19,5346}, {20,5346}, {21,5348}, + {25,5349}, {48,5349}, {56,5589}, {63,5728}, + {64,5994}, {65,6026}, {127,6026}, {128,7875}, + {129,7883}, {160,7900}, {192,8018}, {224,8018}, + {256,8135}, {320,8209}, {384,8333}, {448,8415}, + {512,8572}, {640,8720}, {768,8944}, {896,9093}, + {1024,9379}, {1280,9795}, {1536,10348}, {1792,10999}, + {2048,12274}, {2560,13642}, {3072,14981}, {3584,16649}, + {4096,20263}, {5120,23455}, {6144,25200}, {7168,27963}, + {8192,31800}, {10240,37889}, {12288,43611}, {14336,48748}, + {16384,55631}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_3[] = { + {0,0}, {1,10338}, {2,10448}, {3,10544}, + {4,10647}, {5,10735}, {6,10844}, {8,11034}, + {9,11133}, {10,11248}, {12,11440}, {14,11639}, + {15,11939}, {16,12042}, {17,12141}, {20,12437}, + {23,12741}, {24,12835}, {25,12933}, {27,13138}, + {28,13250}, {30,13434}, {31,13540}, {32,13640}, + {33,13739}, {40,14431}, {48,15236}, {56,16036}, + {63,16733}, {64,16831}, {65,16932}, {80,18435}, + {96,20026}, {112,21622}, {127,23125}, {128,23218}, + {129,23318}, {160,26414}, {192,29630}, {224,32818}, + {256,36004}, {320,43111}, {384,49478}, {448,55891}, + {512,62288}, {640,75878}, {768,88700}, {896,102236}, + {1024,115102}, {1280,141441}, {1536,167820}, {1792,194205}, + {2048,220596}, {2560,273372}, {3072,326212}, {3584,379043}, + {4096,431915}, {5120,537472}, {6144,643342}, {7168,749114}, + {8192,854612}, {10240,1065905}, {12288,1277128}, {14336,1488599}, + {16384,1699794}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_4[] = { + {0,0}, {1,10302}, {2,10406}, {3,10514}, + {4,10608}, {5,10704}, {6,10796}, {7,10902}, + {8,11010}, {9,11098}, {10,11212}, {11,11304}, + {12,11404}, {13,11501}, {14,11590}, {15,11904}, + {16,11999}, {17,12099}, {18,12203}, {19,12301}, + {20,12402}, {21,12498}, {22,12602}, {23,12698}, + {24,12797}, {25,12901}, {26,12997}, {31,13498}, + {33,13694}, {36,13994}, {40,14396}, {48,15187}, + {56,15982}, {63,16685}, {64,16783}, {65,16876}, + {80,18377}, {96,19971}, {112,21564}, {127,23058}, + {128,23161}, {129,23260}, {160,26365}, {192,29551}, + {224,32749}, {256,35949}, {320,43034}, {384,49403}, + {448,55756}, {512,62161}, {640,75704}, {768,88497}, + {896,102015}, {1024,114819}, {1280,141163}, {1536,167471}, + {1792,193803}, {2048,220220}, {2560,272828}, {3072,325725}, + {3584,378498}, {4096,431144}, {5120,536712}, {6144,641979}, + {7168,747539}, {8192,852988}, {10240,1063897}, {12288,1275078}, + {14336,1485584}, {16384,1696059}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_5[] = { + {0,0}, {1,10305}, {2,10398}, {3,10505}, + {4,10605}, {5,10697}, {6,10807}, {7,10898}, + {8,10999}, {9,11097}, {11,11306}, {12,11403}, + {13,11502}, {14,11595}, {15,11894}, {16,12000}, + {17,12093}, {18,12194}, {19,12298}, {20,12400}, + {21,12499}, {22,12594}, {23,12688}, {25,12890}, + {26,12994}, {27,13093}, {28,13189}, {30,13395}, + {31,13489}, {32,13590}, {36,13983}, {40,14388}, + {48,15188}, {56,15984}, {63,16683}, {64,16776}, + {65,16872}, {80,18374}, {96,19968}, {112,21570}, + {127,23057}, {128,23165}, {129,23253}, {160,26358}, + {192,29545}, {224,32717}, {256,35912}, {320,42991}, + {384,49388}, {448,55793}, {512,62157}, {640,75692}, + {768,88483}, {896,102010}, {1024,114790}, {1280,141066}, + {1536,167373}, {1792,193776}, {2048,220049}, {2560,272855}, + {3072,325573}, {3584,378210}, {4096,430980}, {5120,536385}, + {6144,641781}, {7168,747192}, {8192,852597}, {10240,1063400}, + {12288,1274124}, {14336,1484884}, {16384,1695563}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_6[] = { + {0,0}, {2,2154}, {254,2154}, {256,4003}, + {258,4335}, {448,4335}, {512,4388}, {640,4671}, + {768,4836}, {896,4989}, {1024,5189}, {1280,5594}, + {1536,5992}, {1792,6392}, {2048,6795}, {2560,7693}, + {3072,8490}, {3584,9289}, {4096,10101}, {5120,11702}, + {6144,13296}, {7168,14929}, {8192,16534}, {10240,19752}, + {12288,22954}, {14336,26122}, {16384,29922}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_7[] = { + {0,0}, {2,2201}, {46,2202}, {56,2202}, + {60,2300}, {254,2300}, {256,5192}, {258,5207}, + {320,5321}, {384,5416}, {448,5518}, {512,5787}, + {640,5986}, {768,6195}, {896,6393}, {1024,6593}, + {1280,6982}, {1536,7395}, {2048,8193}, {2560,8892}, + {3072,9741}, {3584,10541}, {4096,11360}, {5120,12901}, + {6144,14506}, {7168,16150}, {8192,17744}, {10240,20951}, + {12288,24197}, {14336,27381}, {16384,31241}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_8[] = { + {0,0}, {2,5599}, {4,5599}, {6,5701}, + {8,5701}, {10,5824}, {12,5829}, {14,5893}, + {16,5893}, {18,5967}, {20,5968}, {24,5968}, + {26,5971}, {28,5971}, {30,5984}, {32,5984}, + {34,6022}, {38,6023}, {40,6023}, {42,6034}, + {48,6034}, {50,6044}, {52,6044}, {54,6062}, + {56,6214}, {60,6509}, {62,6569}, {64,6569}, + {66,6679}, {112,6679}, {126,6811}, {128,8587}, + {130,8587}, {160,8598}, {192,8702}, {254,8702}, + {256,8789}, {258,8789}, {320,8895}, {384,9003}, + {448,9094}, {512,9190}, {640,9390}, {896,9786}, + {1024,9994}, {1280,10403}, {1536,11358}, {1792,11815}, + {2048,12477}, {2560,14274}, {3072,15587}, {3584,17319}, + {4096,21061}, {5120,24188}, {6144,25886}, {7168,28620}, + {8192,32952}, {10240,38242}, {12288,44166}, {14336,49301}, + {16384,56734}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_9[] = { + {0,0}, {2,8522}, {4,8620}, {6,8722}, + {10,8924}, {12,9019}, {14,9118}, {16,9222}, + {18,9316}, {20,9424}, {22,9524}, {24,9617}, + {26,9721}, {28,9822}, {30,10123}, {32,10215}, + {34,10318}, {36,10414}, {38,10517}, {40,10615}, + {42,10721}, {44,10824}, {46,10913}, {48,11020}, + {50,11118}, {52,11215}, {54,11309}, {56,11415}, + {60,11618}, {62,11719}, {64,11811}, {66,11928}, + {72,12220}, {80,12618}, {96,13413}, {112,14219}, + {126,14908}, {128,15015}, {130,15106}, {160,16605}, + {192,18216}, {224,19812}, {254,21306}, {256,21406}, + {258,21505}, {320,24598}, {384,27787}, {448,30977}, + {512,34178}, {640,41256}, {768,47629}, {896,54053}, + {1024,60408}, {1280,73966}, {1536,86831}, {1792,100326}, + {2048,113186}, {2560,139506}, {3072,165862}, {3584,192225}, + {4096,218634}, {5120,271501}, {6144,324250}, {7168,376977}, + {8192,429845}, {10240,535661}, {12288,641316}, {14336,746791}, + {16384,852420}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_10[] = { + {0,0}, {2,8479}, {4,8578}, {6,8690}, + {8,8783}, {10,8882}, {12,8977}, {14,9077}, + {18,9283}, {20,9384}, {22,9476}, {24,9575}, + {26,9677}, {28,9775}, {30,10078}, {32,10178}, + {34,10272}, {36,10375}, {38,10474}, {40,10570}, + {42,10674}, {44,10766}, {46,10873}, {48,10971}, + {50,11075}, {52,11172}, {54,11268}, {56,11366}, + {60,11573}, {62,11669}, {64,11768}, {66,11872}, + {72,12172}, {80,12565}, {96,13361}, {112,14162}, + {126,14851}, {128,14964}, {130,15053}, {160,16550}, + {192,18153}, {224,19752}, {254,21242}, {256,21336}, + {258,21436}, {320,24522}, {384,27715}, {448,30898}, + {512,34090}, {640,41143}, {768,47595}, {896,53957}, + {1024,60341}, {1280,73855}, {1536,86673}, {1792,100189}, + {2048,112973}, {2560,139269}, {3072,165646}, {3584,191903}, + {4096,218261}, {5120,271061}, {6144,323668}, {7168,376276}, + {8192,429437}, {10240,534805}, {12288,640044}, {14336,745580}, + {16384,850965}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_11[] = { + {0,0}, {2,8494}, {6,8696}, {8,8796}, + {12,8995}, {14,9092}, {16,9200}, {18,9290}, + {22,9497}, {26,9690}, {28,9791}, {30,10095}, + {32,10200}, {34,10293}, {36,10390}, {38,10482}, + {40,10592}, {42,10690}, {44,10795}, {46,10891}, + {48,10992}, {52,11186}, {54,11287}, {56,11391}, + {60,11591}, {62,11696}, {64,11783}, {66,11890}, + {72,12586}, {80,12979}, {96,13776}, {112,14575}, + {126,15270}, {128,15373}, {130,15472}, {160,16680}, + {192,18273}, {224,19860}, {254,21359}, {256,21451}, + {258,21553}, {320,24647}, {384,28135}, {448,31026}, + {512,34214}, {640,41309}, {768,47738}, {896,54159}, + {1024,60861}, {1280,74099}, {1536,86921}, {1792,100478}, + {2048,113303}, {2560,139602}, {3072,166051}, {3584,192454}, + {4096,219298}, {5120,271749}, {6144,324524}, {7168,377635}, + {8192,430144}, {10240,535960}, {12288,641280}, {14336,746954}, + {16384,852260}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_12[] = { + {0,0}, {4,2007}, {12,2008}, {28,2009}, + {252,2010}, {256,4392}, {448,4392}, {508,4593}, + {512,4593}, {516,4806}, {640,4806}, {768,4885}, + {896,4995}, {1280,5592}, {1536,5991}, {1792,6394}, + {2048,6784}, {2560,7689}, {3072,8486}, {3584,9288}, + {4096,10089}, {5120,11697}, {6144,13281}, {7168,14887}, + {8192,16508}, {10240,19706}, {12288,23699}, {14336,26896}, + {16384,30730}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_13[] = { + {0,0}, {4,2044}, {8,2047}, {36,2047}, + {40,2049}, {56,2049}, {60,2145}, {124,2145}, + {128,2148}, {224,2149}, {252,2149}, {256,5398}, + {260,5407}, {320,5536}, {384,5645}, {448,5725}, + {508,5726}, {512,5795}, {516,5797}, {640,5996}, + {896,6394}, {1024,6592}, {1536,7396}, {1792,7795}, + {2048,8196}, {2560,8900}, {3072,9695}, {3584,10494}, + {4096,11300}, {5120,12913}, {6144,14507}, {7168,16102}, + {8192,17700}, {10240,20956}, {12288,25005}, {14336,28188}, + {16384,31977}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_14[] = { + {0,0}, {4,5886}, {8,5886}, {12,6102}, + {16,6104}, {20,6254}, {40,6254}, {44,6291}, + {48,6291}, {52,6305}, {56,6502}, {60,6768}, + {64,6800}, {68,7011}, {80,7011}, {84,7028}, + {120,7028}, {124,7098}, {128,8884}, {160,8884}, + {192,8976}, {252,8976}, {256,9071}, {260,9071}, + {320,9174}, {384,9279}, {448,9392}, {508,9392}, + {512,9480}, {516,9481}, {640,9673}, {768,9869}, + {896,10069}, {1024,10267}, {1280,10676}, {1536,11632}, + {1792,11871}, {2048,12729}, {2560,14541}, {3072,15853}, + {3584,17752}, {4096,21290}, {5120,24407}, {6144,26114}, + {7168,28779}, {8192,33199}, {10240,38493}, {12288,44405}, + {14336,49511}, {16384,56876}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_15[] = { + {0,0}, {4,8524}, {8,8630}, {12,8724}, + {16,8828}, {20,8927}, {24,9021}, {28,9126}, + {36,9323}, {40,9419}, {44,9522}, {48,9615}, + {52,9720}, {56,9819}, {60,10116}, {64,10221}, + {68,10317}, {76,10522}, {80,10618}, {84,10716}, + {88,10817}, {92,10914}, {96,11017}, {100,11115}, + {104,11216}, {108,11313}, {112,11416}, {120,11608}, + {124,11712}, {128,11810}, {144,12211}, {160,12611}, + {192,13401}, {224,14203}, {252,14899}, {256,15004}, + {260,15102}, {320,16602}, {384,18227}, {448,19814}, + {508,21307}, {512,21401}, {516,21503}, {640,24597}, + {768,27791}, {896,30988}, {1024,34167}, {1280,41270}, + {1536,47660}, {1792,54059}, {2048,60533}, {2560,73957}, + {3072,86888}, {3584,100365}, {4096,113229}, {5120,139552}, + {6144,165735}, {7168,192162}, {8192,218467}, {10240,271046}, + {12288,323774}, {14336,376405}, {16384,429017}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_16[] = { + {0,0}, {4,8504}, {8,8602}, {12,8707}, + {16,8808}, {20,8902}, {24,9004}, {36,9304}, + {40,9401}, {44,9500}, {48,9604}, {52,9706}, + {56,9811}, {60,10103}, {64,10198}, {68,10310}, + {72,10400}, {76,10500}, {80,10604}, {88,10799}, + {96,10998}, {100,11105}, {104,11196}, {112,11396}, + {120,11600}, {124,11698}, {128,11801}, {132,11902}, + {144,12203}, {160,12595}, {224,14191}, {252,14897}, + {256,14987}, {260,15086}, {320,16604}, {384,18184}, + {448,19798}, {508,21295}, {512,21379}, {516,21482}, + {640,24571}, {768,27760}, {896,30957}, {1024,34149}, + {1280,41233}, {1536,47656}, {1792,54092}, {2048,60484}, + {2560,73989}, {3072,86649}, {3584,100123}, {4096,113041}, + {5120,139309}, {6144,165614}, {7168,192090}, {8192,218311}, + {10240,270950}, {12288,323621}, {14336,376640}, {16384,429276}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_17[] = { + {0,0}, {4,8499}, {8,8604}, {24,9000}, + {28,9095}, {32,9193}, {36,9298}, {40,9396}, + {44,9499}, {48,9596}, {52,9699}, {56,9793}, + {60,10094}, {68,10293}, {72,10486}, {76,10589}, + {80,10687}, {84,10790}, {88,10883}, {92,10984}, + {96,11076}, {100,11183}, {104,11284}, {112,11479}, + {124,11781}, {128,11886}, {132,11980}, {144,12579}, + {192,13790}, {224,14286}, {252,14982}, {256,15083}, + {260,15183}, {320,16676}, {384,18260}, {448,19859}, + {508,21346}, {512,21439}, {516,21542}, {640,24628}, + {768,27816}, {896,31052}, {1024,34542}, {1280,41326}, + {1536,47748}, {1792,54145}, {2048,60555}, {2560,74034}, + {3072,86956}, {3584,100460}, {4096,113661}, {5120,139591}, + {6144,165857}, {7168,192137}, {8192,218882}, {10240,270987}, + {12288,323938}, {14336,376629}, {16384,429201}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_18[] = { + {0,0}, {8,2065}, {248,2066}, {256,4223}, + {264,4467}, {448,4467}, {504,4707}, {512,4707}, + {520,4923}, {640,4923}, {768,5010}, {896,5161}, + {1016,5525}, {1024,5525}, {1032,5761}, {1280,5761}, + {1536,6156}, {1792,6553}, {2048,6945}, {2560,7861}, + {3072,8657}, {3584,9452}, {4096,10264}, {5120,11875}, + {6144,13472}, {7168,15090}, {8192,16694}, {10240,19908}, + {12288,23106}, {14336,26316}, {16384,30111}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_19[] = { + {0,0}, {8,2169}, {16,2172}, {56,2172}, + {64,2271}, {144,2272}, {248,2272}, {256,5166}, + {264,5167}, {288,5169}, {320,5283}, {384,5334}, + {504,5334}, {512,5563}, {520,5564}, {640,5764}, + {768,5965}, {896,6024}, {1016,6024}, {1024,6359}, + {1032,6359}, {1280,6757}, {1536,7169}, {1792,7569}, + {2048,7970}, {2560,8672}, {3072,9473}, {3584,10275}, + {4096,11082}, {5120,12688}, {6144,14291}, {7168,15905}, + {8192,17515}, {10240,20762}, {12288,23974}, {14336,27155}, + {16384,30970}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_20[] = { + {0,0}, {8,5751}, {16,5981}, {24,6123}, + {40,6123}, {48,6165}, {56,6413}, {64,6666}, + {72,6745}, {80,6745}, {88,6795}, {120,6795}, + {128,8746}, {144,8746}, {152,8751}, {184,8751}, + {192,8846}, {216,8847}, {224,8850}, {240,8853}, + {248,8853}, {256,8943}, {264,8950}, {288,8952}, + {320,9053}, {384,9149}, {448,9255}, {504,9255}, + {512,9348}, {520,9348}, {640,9537}, {768,9740}, + {896,9936}, {1016,10109}, {1024,10143}, {1032,10346}, + {1280,10546}, {1536,11322}, {1792,11764}, {2048,12823}, + {2560,14460}, {3072,15729}, {3584,17447}, {4096,21199}, + {5120,24324}, {6144,26035}, {7168,28709}, {8192,33091}, + {10240,38355}, {12288,44269}, {14336,49444}, {16384,56842}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_21[] = { + {0,0}, {8,8498}, {16,8599}, {24,8705}, + {32,8794}, {40,8906}, {48,8998}, {56,9096}, + {64,9196}, {72,9299}, {80,9401}, {88,9500}, + {96,9614}, {104,9695}, {112,9795}, {120,10092}, + {128,10192}, {136,10297}, {144,10398}, {152,10498}, + {160,10588}, {168,10698}, {184,10902}, {192,10999}, + {200,11092}, {208,11194}, {216,11293}, {224,11399}, + {240,11589}, {248,11697}, {256,11796}, {264,11907}, + {288,12204}, {320,12592}, {384,13397}, {448,14191}, + {512,14991}, {520,15090}, {640,16592}, {768,18193}, + {896,19792}, {1016,21294}, {1024,21397}, {1032,21493}, + {1280,24565}, {1536,27706}, {1792,30897}, {2048,34125}, + {2560,41129}, {3072,47584}, {3584,53985}, {4096,60373}, + {5120,73895}, {6144,86650}, {7168,100173}, {8192,113034}, + {10240,139305}, {12288,165641}, {14336,191923}, {16384,218268}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_22[] = { + {0,0}, {8,8522}, {24,8721}, {32,8827}, + {40,8924}, {48,9023}, {56,9120}, {64,9220}, + {72,9312}, {80,9420}, {88,9518}, {96,9614}, + {104,9712}, {112,9821}, {120,10119}, {128,10218}, + {136,10324}, {152,10523}, {160,10615}, {168,10722}, + {176,10824}, {184,10931}, {192,11033}, {200,11122}, + {208,11233}, {224,11438}, {240,11634}, {248,11735}, + {256,11826}, {264,11932}, {288,12238}, {320,12638}, + {384,13433}, {448,14230}, {504,14925}, {512,15024}, + {520,15128}, {640,16638}, {768,18244}, {896,19836}, + {1016,21344}, {1024,21441}, {1032,21536}, {1280,24613}, + {1536,27848}, {1792,31032}, {2048,34228}, {2560,41319}, + {3072,47715}, {3584,54072}, {4096,60484}, {5120,74020}, + {6144,86819}, {7168,100391}, {8192,113241}, {10240,139572}, + {12288,165899}, {14336,192230}, {16384,219043}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_23[] = { + {0,0}, {8,8524}, {16,8613}, {24,8710}, + {32,8809}, {40,8910}, {48,9009}, {56,9125}, + {64,9211}, {72,9306}, {80,9406}, {88,9512}, + {96,9621}, {104,9704}, {112,9815}, {120,10199}, + {128,10206}, {136,10314}, {144,10408}, {152,10526}, + {168,10715}, {176,10819}, {184,10927}, {192,11011}, + {200,11123}, {208,11214}, {216,11317}, {224,11412}, + {240,11612}, {248,11710}, {256,11819}, {264,11914}, + {288,12214}, {384,13402}, {448,14207}, {504,14902}, + {512,15007}, {520,15105}, {640,16602}, {768,18204}, + {896,19805}, {1016,21304}, {1024,21408}, {1032,21505}, + {1280,24616}, {1536,27786}, {1792,30979}, {2048,34178}, + {2560,41273}, {3072,47727}, {3584,54201}, {4096,60627}, + {5120,74133}, {6144,87045}, {7168,100632}, {8192,113590}, + {10240,139958}, {12288,166342}, {14336,192740}, {16384,219049}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_24[] = { + {0,0}, {1,542}, {7,542}, {8,567}, + {26,567}, {27,571}, {127,571}, {128,3556}, + {640,3556}, {768,3742}, {1024,4142}, {1280,4740}, + {1536,4938}, {1792,5338}, {2048,5637}, {2560,6631}, + {3072,7626}, {3584,8227}, {4096,9224}, {5120,10722}, + {6144,12417}, {7168,14012}, {8192,15523}, {10240,18718}, + {12288,22726}, {14336,25912}, {16384,28842}, {20480,35232}, + {24576,41609}, {28672,47994}, {32768,55162}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_25[] = { + {0,0}, {1,622}, {2,652}, {21,652}, + {22,670}, {23,670}, {24,681}, {56,681}, + {63,740}, {65,740}, {80,789}, {127,789}, + {128,3802}, {160,3802}, {192,3902}, {224,3903}, + {256,4104}, {320,4199}, {384,4199}, {448,4289}, + {512,4299}, {640,4487}, {768,5084}, {896,5185}, + {1024,5285}, {1280,5698}, {1536,5885}, {1792,6684}, + {2048,6904}, {2560,7589}, {3072,8585}, {3584,9185}, + {4096,10101}, {5120,11805}, {6144,13413}, {7168,15014}, + {8192,16613}, {10240,19831}, {12288,23893}, {14336,27094}, + {16384,29901}, {20480,36314}, {24576,42685}, {28672,49090}, + {32768,56125}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_26[] = { + {0,0}, {1,648}, {2,651}, {5,651}, + {6,659}, {7,659}, {8,667}, {27,667}, + {28,671}, {31,671}, {32,687}, {33,734}, + {40,734}, {48,743}, {65,743}, {80,760}, + {96,760}, {112,786}, {127,786}, {128,3790}, + {160,3790}, {192,3888}, {224,3888}, {256,4093}, + {320,4197}, {384,4201}, {448,4280}, {512,4280}, + {640,4478}, {768,5084}, {896,5178}, {1024,5291}, + {1280,5679}, {1536,5880}, {1792,6680}, {2048,6882}, + {2560,7582}, {3072,8581}, {3584,9177}, {4096,10085}, + {5120,11793}, {6144,13413}, {7168,15021}, {8192,16616}, + {10240,19823}, {12288,23878}, {14336,27077}, {16384,29844}, + {20480,36260}, {24576,42672}, {28672,49077}, {32768,56161}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_27[] = { + {0,0}, {1,10057}, {2,10224}, {3,10325}, + {4,10423}, {5,10524}, {6,10620}, {7,10721}, + {8,10826}, {9,10919}, {10,11011}, {11,11125}, + {12,11195}, {13,11340}, {14,11414}, {15,11724}, + {17,11921}, {18,12023}, {20,12224}, {21,12322}, + {22,12419}, {24,12634}, {25,12720}, {26,12813}, + {27,12928}, {28,13036}, {30,13215}, {31,13334}, + {32,13423}, {33,13521}, {40,14228}, {48,15018}, + {56,15820}, {63,16513}, {64,16628}, {65,16719}, + {80,18231}, {96,19830}, {112,21445}, {127,22977}, + {128,23052}, {129,23141}, {160,26256}, {192,29457}, + {224,32657}, {256,35871}, {320,42472}, {384,48877}, + {448,55284}, {512,61701}, {640,74796}, {768,87570}, + {896,100642}, {1024,113495}, {1280,139296}, {1536,165156}, + {1792,191049}, {2048,216943}, {2560,268612}, {3072,320349}, + {3584,371936}, {4096,423762}, {5120,527397}, {6144,630805}, + {7168,734233}, {8192,837783}, {10240,1045026}, {12288,1252484}, + {14336,1458908}, {16384,1665863}, {20480,2080238}, {24576,2494174}, + {28672,2908097}, {32768,3321957}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_28[] = { + {0,0}, {1,10075}, {2,10197}, {3,10301}, + {4,10402}, {5,10502}, {6,10609}, {7,10702}, + {8,10802}, {9,10897}, {10,10998}, {11,11083}, + {12,11197}, {13,11284}, {14,11375}, {15,11695}, + {16,11804}, {17,11900}, {18,12003}, {19,12100}, + {20,12199}, {21,12296}, {22,12400}, {23,12494}, + {24,12594}, {25,12696}, {26,12812}, {27,12877}, + {28,13010}, {30,13202}, {31,13285}, {32,13397}, + {33,13498}, {36,13795}, {40,14149}, {48,14996}, + {56,15825}, {63,16483}, {64,16613}, {65,16699}, + {80,18208}, {96,19798}, {112,21405}, {127,22888}, + {128,23009}, {129,23107}, {160,26211}, {192,29409}, + {224,32599}, {256,35795}, {320,42394}, {384,48789}, + {448,55192}, {512,61590}, {640,74589}, {768,87432}, + {896,100426}, {1024,113292}, {1280,139051}, {1536,164845}, + {1792,190635}, {2048,216526}, {2560,268124}, {3072,319861}, + {3584,371367}, {4096,423112}, {5120,526462}, {6144,629686}, + {7168,733060}, {8192,836658}, {10240,1042878}, {12288,1250008}, + {14336,1456665}, {16384,1664023}, {20480,2076507}, {24576,2491260}, + {28672,2904375}, {32768,3316349}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_29[] = { + {0,0}, {1,10089}, {2,10207}, {3,10301}, + {4,10405}, {5,10504}, {7,10698}, {9,10899}, + {10,10983}, {11,11119}, {12,11192}, {13,11291}, + {14,11417}, {15,11698}, {16,11802}, {18,12000}, + {19,12096}, {20,12198}, {21,12298}, {22,12395}, + {24,12605}, {25,12714}, {26,12787}, {27,12910}, + {28,13010}, {30,13197}, {31,13329}, {32,13401}, + {33,13498}, {36,13800}, {40,14188}, {48,14995}, + {56,15793}, {63,16501}, {64,16592}, {65,16696}, + {80,18191}, {96,19798}, {112,21397}, {127,22887}, + {128,23006}, {129,23106}, {160,26211}, {192,29410}, + {224,32606}, {256,35805}, {320,42389}, {384,48791}, + {448,55197}, {512,61572}, {640,74626}, {768,87399}, + {896,100441}, {1024,113274}, {1280,139058}, {1536,164839}, + {1792,190736}, {2048,216548}, {2560,268110}, {3072,319767}, + {3584,371365}, {4096,423010}, {5120,526400}, {6144,629547}, + {7168,732994}, {8192,836239}, {10240,1042988}, {12288,1249754}, + {14336,1456154}, {16384,1663029}, {20480,2077570}, {24576,2489258}, + {28672,2903666}, {32768,3316914}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_30[] = { + {0,0}, {2,1636}, {20,1636}, {22,1660}, + {64,1660}, {66,1680}, {96,1680}, {112,1719}, + {126,1725}, {128,4031}, {130,4203}, {192,4203}, + {224,4276}, {256,4276}, {258,4350}, {512,4350}, + {640,4429}, {768,4727}, {896,4828}, {1024,5028}, + {1280,5724}, {1536,5925}, {1792,6275}, {2048,6624}, + {2560,7622}, {3072,8621}, {3584,9221}, {4096,10120}, + {5120,11738}, {6144,13336}, {8192,16528}, {10240,19729}, + {12288,23773}, {14336,26976}, {16384,29869}, {20480,36266}, + {24576,42697}, {28672,49107}, {32768,56299}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_31[] = { + {0,0}, {2,1727}, {4,1729}, {56,1729}, + {60,1829}, {126,1830}, {128,4723}, {130,4729}, + {160,4737}, {192,4836}, {224,4837}, {254,4837}, + {256,5035}, {258,5038}, {320,5140}, {384,5144}, + {448,5224}, {512,5224}, {640,5427}, {768,6018}, + {896,6125}, {1024,6219}, {1280,6626}, {1536,6927}, + {1792,7624}, {2048,7824}, {2560,8524}, {3072,9521}, + {3584,10220}, {4096,11211}, {5120,12765}, {6144,14362}, + {7168,15996}, {8192,17548}, {10240,20699}, {12288,24857}, + {14336,28021}, {16384,30757}, {20480,37216}, {28672,50000}, + {32768,56947}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_32[] = { + {0,0}, {2,1741}, {32,1741}, {34,1832}, + {62,1832}, {64,1840}, {66,1841}, {112,1842}, + {126,1842}, {128,4675}, {160,4675}, {192,4742}, + {224,4742}, {254,4846}, {256,5039}, {258,5039}, + {320,5139}, {384,5139}, {448,5233}, {512,5233}, + {640,5434}, {768,6032}, {896,6129}, {1024,6230}, + {1280,6629}, {1536,6937}, {1792,7629}, {2048,7828}, + {2560,8526}, {3072,9527}, {3584,10127}, {4096,12778}, + {5120,14376}, {6144,16102}, {7168,17679}, {8192,21361}, + {10240,25421}, {12288,30830}, {14336,34024}, {16384,39133}, + {20480,47971}, {24576,56570}, {28672,65287}, {32768,74010}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_33[] = { + {0,0}, {2,8403}, {4,8498}, {6,8602}, + {8,8696}, {10,8802}, {18,9197}, {20,9302}, + {22,9401}, {24,9492}, {26,9598}, {28,9699}, + {30,10000}, {34,10195}, {36,10296}, {38,10399}, + {40,10498}, {42,10601}, {48,10900}, {50,10996}, + {52,11100}, {54,11197}, {56,11297}, {60,11492}, + {62,11596}, {72,12095}, {80,12496}, {96,13294}, + {112,14095}, {126,14791}, {128,14894}, {130,14995}, + {160,16504}, {192,18115}, {224,19716}, {254,21213}, + {256,21314}, {258,21409}, {320,24512}, {384,27721}, + {448,30925}, {512,34115}, {640,40725}, {768,47100}, + {896,53503}, {1024,59900}, {1280,72902}, {1536,85713}, + {1792,98730}, {2048,111561}, {2560,137486}, {3072,163310}, + {3584,189193}, {4096,214983}, {6144,318335}, {7168,370023}, + {8192,422036}, {10240,525461}, {12288,628916}, {14336,732312}, + {16384,835717}, {20480,1042630}, {24576,1249444}, {28672,1456377}, + {32768,1663133}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_34[] = { + {0,0}, {2,8399}, {6,8602}, {8,8700}, + {10,8801}, {14,8996}, {16,9095}, {18,9198}, + {20,9292}, {22,9392}, {24,9497}, {26,9597}, + {28,9693}, {30,9992}, {32,10097}, {36,10296}, + {38,10397}, {40,10502}, {42,10595}, {44,10696}, + {48,10897}, {50,11000}, {52,11092}, {54,11199}, + {56,11299}, {60,11497}, {62,11602}, {64,11692}, + {66,11793}, {72,12093}, {96,13287}, {112,14092}, + {126,14794}, {128,14886}, {130,14985}, {160,16498}, + {192,18094}, {224,19704}, {254,21205}, {256,21309}, + {258,21405}, {320,24498}, {384,27702}, {448,30905}, + {512,34102}, {640,40670}, {768,47110}, {896,53461}, + {1024,59856}, {1280,72846}, {1536,85685}, {1792,98689}, + {2048,111548}, {2560,137377}, {3072,163168}, {3584,188993}, + {4096,214795}, {5120,266428}, {6144,318040}, {7168,369674}, + {8192,421298}, {10240,524714}, {12288,628044}, {14336,731658}, + {16384,834762}, {20480,1041900}, {24576,1247960}, {28672,1455296}, + {32768,1661373}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_35[] = { + {0,0}, {2,8419}, {4,8515}, {6,8610}, + {12,8914}, {14,9012}, {18,9213}, {20,9319}, + {22,9408}, {24,9514}, {26,9613}, {28,9720}, + {30,10010}, {32,10116}, {34,10206}, {36,10316}, + {38,10409}, {40,10514}, {42,10613}, {44,10710}, + {46,10810}, {48,10914}, {50,11012}, {52,11118}, + {54,11216}, {56,11310}, {60,11515}, {62,11611}, + {64,11706}, {66,11810}, {80,12516}, {96,13313}, + {112,14117}, {126,14815}, {128,14928}, {130,15027}, + {160,16520}, {192,18125}, {224,19729}, {254,21222}, + {256,21325}, {258,21423}, {320,24529}, {384,27729}, + {448,30932}, {512,34136}, {640,40740}, {768,47108}, + {896,53512}, {1024,59915}, {1280,72930}, {1536,85805}, + {1792,98825}, {2048,111649}, {2560,137503}, {3072,163321}, + {3584,189171}, {4096,215002}, {5120,266638}, {6144,318372}, + {7168,370198}, {8192,421862}, {10240,525320}, {12288,628941}, + {14336,732319}, {16384,836029}, {20480,1042839}, {24576,1249510}, + {28672,1456221}, {32768,1664069}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_36[] = { + {0,0}, {4,1677}, {124,1677}, {128,4089}, + {132,4263}, {192,4263}, {224,4321}, {256,4321}, + {260,4400}, {448,4400}, {508,4571}, {512,4571}, + {516,4637}, {640,4637}, {768,4771}, {896,4871}, + {1024,5069}, {1280,5770}, {1536,5965}, {1792,6319}, + {2048,6670}, {2560,7663}, {3072,8661}, {3584,9266}, + {4096,10165}, {5120,11765}, {6144,13368}, {7168,14987}, + {8192,16590}, {10240,19791}, {12288,22985}, {14336,26190}, + {16384,29100}, {20480,35493}, {24576,41900}, {28672,48309}, + {32768,55520}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_37[] = { + {0,0}, {4,1780}, {32,1781}, {56,1781}, + {60,1875}, {120,1875}, {124,1879}, {128,4785}, + {144,4785}, {160,4789}, {192,4885}, {252,4885}, + {256,5084}, {260,5086}, {320,5189}, {384,5190}, + {448,5273}, {508,5273}, {512,5276}, {516,5276}, + {640,5469}, {768,6075}, {896,6171}, {1024,6269}, + {1280,6675}, {1536,6967}, {1792,7666}, {2048,7868}, + {2560,8567}, {3072,9567}, {3584,10266}, {4096,11064}, + {5120,12811}, {6144,14364}, {7168,15968}, {8192,17650}, + {10240,20809}, {12288,24093}, {14336,27204}, {16384,29996}, + {20480,36382}, {24576,42802}, {28672,49191}, {32768,56219}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_38[] = { + {0,0}, {4,1799}, {20,1800}, {32,1800}, + {36,1890}, {60,1890}, {64,1894}, {68,1895}, + {88,1896}, {104,1897}, {124,1897}, {128,4751}, + {160,4751}, {192,4801}, {224,4801}, {252,4901}, + {256,5101}, {260,5101}, {320,5198}, {384,5198}, + {448,5286}, {512,5286}, {516,5395}, {640,5487}, + {768,6093}, {896,6185}, {1024,6285}, {1280,6686}, + {1536,6986}, {1792,7685}, {2048,7887}, {2560,8587}, + {3072,9585}, {3584,10186}, {4096,12837}, {5120,14551}, + {6144,16152}, {7168,17716}, {8192,21419}, {10240,24620}, + {12288,30021}, {14336,33336}, {16384,38434}, {20480,47051}, + {24576,55742}, {28672,64472}, {32768,73232}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_39[] = { + {0,0}, {4,8521}, {8,8625}, {12,8725}, + {16,8828}, {20,8919}, {24,9026}, {28,9125}, + {36,9318}, {40,9420}, {44,9525}, {48,9622}, + {52,9722}, {56,9815}, {60,10117}, {76,10516}, + {84,10724}, {88,10823}, {92,10918}, {96,11018}, + {100,11121}, {104,11220}, {108,11326}, {112,11420}, + {120,11624}, {132,11925}, {144,12227}, {160,12620}, + {192,13426}, {224,14221}, {252,14923}, {256,15021}, + {260,15122}, {320,16630}, {384,18246}, {448,19854}, + {508,21342}, {512,21431}, {516,21556}, {640,24647}, + {768,27862}, {896,31062}, {1024,34266}, {1280,40880}, + {1536,47302}, {1792,53709}, {2048,60125}, {2560,73132}, + {3072,85990}, {3584,99015}, {4096,111811}, {5120,137517}, + {6144,163347}, {7168,189205}, {8192,215035}, {10240,266715}, + {12288,318419}, {14336,369992}, {16384,421559}, {20480,524783}, + {24576,628114}, {28672,731967}, {32768,835101}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_40[] = { + {0,0}, {4,8536}, {8,8627}, {12,8725}, + {20,8932}, {24,9027}, {28,9135}, {32,9232}, + {36,9334}, {40,9425}, {44,9533}, {48,9631}, + {52,9728}, {56,9829}, {60,10120}, {64,10228}, + {68,10330}, {72,10427}, {76,10530}, {80,10630}, + {84,10725}, {88,10827}, {92,10928}, {96,11035}, + {100,11127}, {104,11226}, {108,11331}, {112,11431}, + {120,11623}, {124,11729}, {128,11824}, {132,11929}, + {144,12223}, {160,12625}, {192,13423}, {224,14229}, + {252,14922}, {256,15020}, {260,15127}, {320,16624}, + {384,18218}, {448,19814}, {508,21313}, {512,21428}, + {516,21511}, {640,24635}, {768,27846}, {896,31055}, + {1024,34257}, {1280,40863}, {1536,47274}, {1792,53690}, + {2048,60093}, {2560,73114}, {3072,85919}, {3584,98770}, + {4096,111748}, {5120,137426}, {6144,163330}, {7168,189112}, + {8192,214878}, {10240,266476}, {12288,318179}, {14336,369711}, + {16384,421390}, {20480,524392}, {24576,628086}, {28672,730782}, + {32768,834688}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_41[] = { + {0,0}, {4,8521}, {8,8618}, {16,8819}, + {20,8922}, {24,9017}, {28,9120}, {32,9221}, + {36,9318}, {40,9417}, {44,9524}, {48,9622}, + {52,9717}, {56,9822}, {60,10118}, {64,10222}, + {68,10325}, {72,10417}, {76,10524}, {80,10620}, + {84,10718}, {88,10815}, {92,10922}, {96,11019}, + {100,11122}, {108,11318}, {112,11423}, {120,11621}, + {124,11715}, {128,11823}, {132,11917}, {144,12216}, + {160,12617}, {192,13420}, {224,14217}, {252,14910}, + {256,15017}, {320,16618}, {384,18233}, {448,19816}, + {508,21325}, {512,21418}, {516,21525}, {640,24624}, + {768,27854}, {896,31060}, {1024,34272}, {1280,40892}, + {1536,47302}, {1792,53702}, {2048,60132}, {2560,73136}, + {3072,85838}, {3584,98883}, {4096,111685}, {5120,137399}, + {6144,163236}, {7168,189207}, {8192,215042}, {10240,266607}, + {12288,318338}, {14336,369897}, {16384,421616}, {20480,524800}, + {24576,628148}, {28672,731393}, {32768,834737}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_42[] = { + {0,0}, {8,1616}, {120,1616}, {128,4054}, + {136,4243}, {192,4243}, {200,4326}, {256,4326}, + {264,4408}, {448,4408}, {504,4540}, {512,4540}, + {520,4604}, {640,4604}, {768,4738}, {896,4838}, + {1016,5389}, {1024,5389}, {1032,5738}, {1280,5738}, + {1536,5934}, {1792,6287}, {2048,6639}, {2560,7638}, + {3072,8638}, {3584,9238}, {4096,10136}, {5120,11739}, + {6144,13336}, {7168,14938}, {8192,16542}, {10240,19768}, + {12288,22970}, {14336,26141}, {16384,29041}, {20480,35483}, + {24576,41884}, {28672,48288}, {32768,55539}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_43[] = { + {0,0}, {8,1742}, {48,1743}, {56,1743}, + {64,1828}, {120,1828}, {128,4721}, {184,4721}, + {192,4817}, {248,4817}, {256,5043}, {264,5045}, + {288,5048}, {320,5105}, {504,5105}, {512,5231}, + {520,5231}, {640,5431}, {768,5918}, {1016,5918}, + {1024,6229}, {1032,6229}, {1280,6629}, {1536,6928}, + {1792,7627}, {2048,7835}, {2560,8521}, {3072,9528}, + {3584,10221}, {4096,11229}, {5120,12768}, {6144,14319}, + {7168,16013}, {8192,17569}, {10240,20719}, {12288,24015}, + {14336,27120}, {16384,29910}, {20480,36299}, {24576,42683}, + {28672,49146}, {32768,56144}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_44[] = { + {0,0}, {8,1746}, {32,1746}, {40,1829}, + {56,1829}, {64,1842}, {120,1842}, {128,4712}, + {160,4712}, {168,4750}, {184,4751}, {192,4810}, + {224,4810}, {240,4847}, {248,4850}, {256,4928}, + {288,4928}, {320,5146}, {384,5146}, {448,5235}, + {504,5240}, {520,5240}, {640,5434}, {768,6033}, + {896,6135}, {1016,6235}, {1032,6235}, {1280,6633}, + {1536,6935}, {1792,7634}, {2048,7847}, {2560,8533}, + {3072,9533}, {3584,10265}, {4096,12877}, {5120,14392}, + {6144,16094}, {7168,17667}, {8192,21379}, {10240,24584}, + {12288,29987}, {14336,33393}, {16384,38291}, {20480,46995}, + {24576,55708}, {28672,64413}, {32768,73121}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_45[] = { + {0,0}, {8,8423}, {24,8615}, {32,8716}, + {40,8819}, {48,8920}, {56,9020}, {64,9118}, + {72,9220}, {80,9315}, {88,9423}, {96,9515}, + {104,9621}, {112,9718}, {120,10018}, {128,10117}, + {136,10214}, {144,10315}, {152,10424}, {160,10526}, + {168,10618}, {176,10726}, {184,10823}, {200,11021}, + {208,11124}, {216,11222}, {224,11324}, {240,11515}, + {248,11615}, {256,11713}, {264,11760}, {288,12102}, + {320,12516}, {384,13319}, {448,14125}, {504,14823}, + {512,14921}, {520,15028}, {640,16534}, {768,18118}, + {896,19727}, {1016,21274}, {1024,21335}, {1032,21433}, + {1280,24569}, {1536,27704}, {1792,30902}, {2048,34107}, + {2560,40703}, {3584,53497}, {4096,59890}, {5120,72887}, + {6144,85684}, {7168,98684}, {8192,111481}, {10240,137276}, + {12288,163265}, {14336,188852}, {16384,214637}, {20480,266532}, + {24576,318215}, {28672,369776}, {32768,421452}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_46[] = { + {0,0}, {8,8428}, {16,8534}, {24,8636}, + {40,8827}, {48,8931}, {64,9133}, {72,9236}, + {80,9336}, {88,9432}, {96,9539}, {104,9626}, + {112,9736}, {120,10032}, {128,10126}, {136,10234}, + {144,10326}, {152,10431}, {160,10529}, {168,10631}, + {176,10738}, {184,10830}, {200,11033}, {208,11128}, + {216,11230}, {224,11334}, {240,11534}, {248,11625}, + {256,11731}, {264,11829}, {288,12133}, {320,12527}, + {384,13324}, {448,14126}, {504,14837}, {512,14925}, + {520,15032}, {640,16537}, {768,18122}, {896,19751}, + {1016,21234}, {1024,21374}, {1032,21434}, {1280,24579}, + {1536,27723}, {1792,30967}, {2048,34090}, {2560,40682}, + {3072,47077}, {3584,53462}, {4096,59856}, {5120,72832}, + {6144,85624}, {7168,98617}, {8192,111398}, {10240,137169}, + {12288,162978}, {14336,188708}, {16384,214756}, {20480,266308}, + {24576,317993}, {28672,369536}, {32768,421109}, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_47[] = { + {0,0}, {8,8420}, {16,8518}, {24,8615}, + {32,8715}, {40,8817}, {56,9019}, {64,9118}, + {72,9221}, {80,9322}, {88,9418}, {96,9521}, + {104,9622}, {112,9725}, {120,10020}, {128,10123}, + {144,10320}, {152,10423}, {160,10520}, {168,10622}, + {176,10722}, {184,10821}, {192,10923}, {200,11024}, + {208,11121}, {216,11216}, {224,11315}, {240,11521}, + {248,11617}, {256,11716}, {264,11828}, {288,12122}, + {320,12520}, {384,13315}, {448,14114}, {504,14821}, + {512,14920}, {520,15015}, {640,16517}, {768,18135}, + {896,19730}, {1016,21268}, {1032,21436}, {1280,24576}, + {1536,27757}, {1792,30892}, {2048,34088}, {2560,40709}, + {3072,47071}, {3584,53468}, {4096,59866}, {5120,72850}, + {6144,85648}, {7168,98765}, {8192,111536}, {10240,137373}, + {12288,163196}, {14336,189002}, {16384,214787}, {20480,266432}, + {24576,318093}, {28672,369712}, {32768,421378}, +}; + +static const ClgRepBulkCurve clg_rep_bulk_curves[] = { + { { CLG_REP_OP_MOVS, 1u, 0u, 0u }, clg_rep_bulk_pts_0, 29u }, + { { CLG_REP_OP_MOVS, 1u, 0u, 1u }, clg_rep_bulk_pts_1, 34u }, + { { CLG_REP_OP_MOVS, 1u, 0u, 2u }, clg_rep_bulk_pts_2, 53u }, + { { CLG_REP_OP_MOVS, 1u, 1u, 0u }, clg_rep_bulk_pts_3, 65u }, + { { CLG_REP_OP_MOVS, 1u, 1u, 1u }, clg_rep_bulk_pts_4, 70u }, + { { CLG_REP_OP_MOVS, 1u, 1u, 2u }, clg_rep_bulk_pts_5, 71u }, + { { CLG_REP_OP_MOVS, 2u, 0u, 0u }, clg_rep_bulk_pts_6, 27u }, + { { CLG_REP_OP_MOVS, 2u, 0u, 1u }, clg_rep_bulk_pts_7, 31u }, + { { CLG_REP_OP_MOVS, 2u, 0u, 2u }, clg_rep_bulk_pts_8, 61u }, + { { CLG_REP_OP_MOVS, 2u, 1u, 0u }, clg_rep_bulk_pts_9, 69u }, + { { CLG_REP_OP_MOVS, 2u, 1u, 1u }, clg_rep_bulk_pts_10, 69u }, + { { CLG_REP_OP_MOVS, 2u, 1u, 2u }, clg_rep_bulk_pts_11, 65u }, + { { CLG_REP_OP_MOVS, 4u, 0u, 0u }, clg_rep_bulk_pts_12, 29u }, + { { CLG_REP_OP_MOVS, 4u, 0u, 1u }, clg_rep_bulk_pts_13, 37u }, + { { CLG_REP_OP_MOVS, 4u, 0u, 2u }, clg_rep_bulk_pts_14, 50u }, + { { CLG_REP_OP_MOVS, 4u, 1u, 0u }, clg_rep_bulk_pts_15, 63u }, + { { CLG_REP_OP_MOVS, 4u, 1u, 1u }, clg_rep_bulk_pts_16, 60u }, + { { CLG_REP_OP_MOVS, 4u, 1u, 2u }, clg_rep_bulk_pts_17, 59u }, + { { CLG_REP_OP_MOVS, 8u, 0u, 0u }, clg_rep_bulk_pts_18, 31u }, + { { CLG_REP_OP_MOVS, 8u, 0u, 1u }, clg_rep_bulk_pts_19, 37u }, + { { CLG_REP_OP_MOVS, 8u, 0u, 2u }, clg_rep_bulk_pts_20, 52u }, + { { CLG_REP_OP_MOVS, 8u, 1u, 0u }, clg_rep_bulk_pts_21, 60u }, + { { CLG_REP_OP_MOVS, 8u, 1u, 1u }, clg_rep_bulk_pts_22, 59u }, + { { CLG_REP_OP_MOVS, 8u, 1u, 2u }, clg_rep_bulk_pts_23, 60u }, + { { CLG_REP_OP_STOS, 1u, 0u, 0u }, clg_rep_bulk_pts_24, 31u }, + { { CLG_REP_OP_STOS, 1u, 0u, 1u }, clg_rep_bulk_pts_25, 45u }, + { { CLG_REP_OP_STOS, 1u, 0u, 2u }, clg_rep_bulk_pts_26, 52u }, + { { CLG_REP_OP_STOS, 1u, 1u, 0u }, clg_rep_bulk_pts_27, 74u }, + { { CLG_REP_OP_STOS, 1u, 1u, 1u }, clg_rep_bulk_pts_28, 78u }, + { { CLG_REP_OP_STOS, 1u, 1u, 2u }, clg_rep_bulk_pts_29, 74u }, + { { CLG_REP_OP_STOS, 2u, 0u, 0u }, clg_rep_bulk_pts_30, 39u }, + { { CLG_REP_OP_STOS, 2u, 0u, 1u }, clg_rep_bulk_pts_31, 41u }, + { { CLG_REP_OP_STOS, 2u, 0u, 2u }, clg_rep_bulk_pts_32, 44u }, + { { CLG_REP_OP_STOS, 2u, 1u, 0u }, clg_rep_bulk_pts_33, 65u }, + { { CLG_REP_OP_STOS, 2u, 1u, 1u }, clg_rep_bulk_pts_34, 69u }, + { { CLG_REP_OP_STOS, 2u, 1u, 2u }, clg_rep_bulk_pts_35, 70u }, + { { CLG_REP_OP_STOS, 4u, 0u, 0u }, clg_rep_bulk_pts_36, 37u }, + { { CLG_REP_OP_STOS, 4u, 0u, 1u }, clg_rep_bulk_pts_37, 44u }, + { { CLG_REP_OP_STOS, 4u, 0u, 2u }, clg_rep_bulk_pts_38, 47u }, + { { CLG_REP_OP_STOS, 4u, 1u, 0u }, clg_rep_bulk_pts_39, 63u }, + { { CLG_REP_OP_STOS, 4u, 1u, 1u }, clg_rep_bulk_pts_40, 69u }, + { { CLG_REP_OP_STOS, 4u, 1u, 2u }, clg_rep_bulk_pts_41, 67u }, + { { CLG_REP_OP_STOS, 8u, 0u, 0u }, clg_rep_bulk_pts_42, 39u }, + { { CLG_REP_OP_STOS, 8u, 0u, 1u }, clg_rep_bulk_pts_43, 42u }, + { { CLG_REP_OP_STOS, 8u, 0u, 2u }, clg_rep_bulk_pts_44, 47u }, + { { CLG_REP_OP_STOS, 8u, 1u, 0u }, clg_rep_bulk_pts_45, 63u }, + { { CLG_REP_OP_STOS, 8u, 1u, 1u }, clg_rep_bulk_pts_46, 63u }, + { { CLG_REP_OP_STOS, 8u, 1u, 2u }, clg_rep_bulk_pts_47, 63u }, +}; diff --git a/callgrind/sim.c b/callgrind/sim.c index db3f9bef18..463438da9d 100644 --- a/callgrind/sim.c +++ b/callgrind/sim.c @@ -29,6 +29,7 @@ */ #include "global.h" +#include "rep_bulk.h" /* Notes: @@ -1181,8 +1182,12 @@ static void log_1I1Dr(InstrInfo* ii, Addr data_addr, Word data_size) inc_costs(IrRes, cost_Ir, CLG_(current_state).cost + fullOffset(EG_IR) ); - inc_costs(DrRes, cost_Dr, - CLG_(current_state).cost + fullOffset(EG_DR) ); + /* REP bulk model: the curve replaces L1-hit refs, and an L1 miss that + * hits LL streams at near-hit cost in a REP copy, so only accesses + * that miss LL are reported. */ + if (!ii->rep_bulk_op || DrRes == MemAccess) + inc_costs(DrRes, cost_Dr, + CLG_(current_state).cost + fullOffset(EG_DR) ); } } @@ -1209,8 +1214,9 @@ static void log_0I1Dr(InstrInfo* ii, Addr data_addr, Word data_size) else cost_Dr = CLG_(cost_base) + ii->cost_offset + ii->eventset->offset[EG_DR]; - inc_costs(DrRes, cost_Dr, - CLG_(current_state).cost + fullOffset(EG_DR) ); + if (!ii->rep_bulk_op || DrRes == MemAccess) + inc_costs(DrRes, cost_Dr, + CLG_(current_state).cost + fullOffset(EG_DR) ); } } @@ -1244,8 +1250,9 @@ static void log_1I1Dw(InstrInfo* ii, Addr data_addr, Word data_size) inc_costs(IrRes, cost_Ir, CLG_(current_state).cost + fullOffset(EG_IR) ); - inc_costs(DwRes, cost_Dw, - CLG_(current_state).cost + fullOffset(EG_DW) ); + if (!ii->rep_bulk_op || DwRes == MemAccess) + inc_costs(DwRes, cost_Dw, + CLG_(current_state).cost + fullOffset(EG_DW) ); } } @@ -1269,13 +1276,79 @@ static void log_0I1Dw(InstrInfo* ii, Addr data_addr, Word data_size) else cost_Dw = CLG_(cost_base) + ii->cost_offset + ii->eventset->offset[EG_DW]; - inc_costs(DwRes, cost_Dw, - CLG_(current_state).cost + fullOffset(EG_DW) ); + if (!ii->rep_bulk_op || DwRes == MemAccess) + inc_costs(DwRes, cost_Dw, + CLG_(current_state).cost + fullOffset(EG_DW) ); } } +/*------------------------------------------------------------*/ +/*--- REP bulk cost model helper ---*/ +/*------------------------------------------------------------*/ + +/* Alignment class from the pointers of the operation's first element (see + * rep_bulk.h). STOS and merged read-modify events pass src == dst. */ +static unsigned rep_align_class(Addr src, Addr dst) +{ + UInt so = (UInt)(src & 63); + UInt dofs = (UInt)(dst & 63); + + if (so == 0 && dofs == 0) + return CLG_REP_ALIGN_ALIGNED; + if (so == 8 && dofs == 8) + return CLG_REP_ALIGN_SAME_UNALIGN; + return CLG_REP_ALIGN_OTHER; +} + +/* `rem` is the repeat count before VEX decrements it, so 0 means the + * instruction finishes without another element. The curve delta is charged + * to both Ct and Cl, so the consumer's sqrt(Ct*Cl) recovers it. */ +void CLG_(log_rep_bulk)(InstrInfo* ii, ULong rem, Addr src, Addr dst, + Word dflag) +{ + exec_state* es = &CLG_(current_state); + unsigned op = ii->rep_bulk_op; + unsigned width = ii->rep_elem_width; + unsigned dir = ((Long)dflag < 0) ? CLG_REP_DIR_BACKWARD + : CLG_REP_DIR_FORWARD; + ULong delta; + ULong * cost_C, *global_C; + + if (rem == 0) + return; + + /* The alignment class is latched from the operation's first element, so + * address changes between elements cannot switch curves. The next + * element continues iff it reports the count and destination expected + * after the previous one; the instruction site cannot be compared since + * VEX translates the first element as part of the preceding block. */ + if (es->rep_next_rem != rem || es->rep_next_dst != dst) + es->rep_align = rep_align_class(src, dst); + es->rep_next_rem = rem - 1; + es->rep_next_dst = dir == CLG_REP_DIR_BACKWARD ? dst - width : dst + width; + + if (!es->collect) + return; + + delta = clg_rep_bulk_marginal(op, width, dir, es->rep_align, + rem * (ULong)width, (ULong)width); + + if (es->nonskipped) + cost_C = es->nonskipped->skipped + fullOffset(EG_CYCLES); + else + cost_C = CLG_(cost_base) + ii->cost_offset + + ii->eventset->offset[EG_CYCLES]; + global_C = es->cost + fullOffset(EG_CYCLES); + + cost_C[0] += delta; + cost_C[1] += delta; + global_C[0] += delta; + global_C[1] += delta; +} + + /*------------------------------------------------------------*/ /*--- Cache configuration ---*/ /*------------------------------------------------------------*/ @@ -1290,6 +1363,21 @@ static void cachesim_post_clo_init(void) /* Cache configurations. */ cache_t I1c, D1c, LLc; + /* REP bulk cost model (on by default). It needs real cache simulation and + * the per-instruction cycle model it replaces, and is currently amd64-only + * because it reads the repeat counter and DF from the guest state. In any + * other mode it stays inert, so prior behavior is unchanged. */ +#if defined(VGA_amd64) + CLG_(clo).rep_bulk_cost = CLG_(clo).rep_bulk_cost + && CLG_(clo).simulate_cache + && CLG_(clo).cycle_estimation + && !clo_simulate_writeback && !clo_collect_cacheuse; +#else + CLG_(clo).rep_bulk_cost = False; +#endif + if (CLG_(clo).rep_bulk_cost) + clg_rep_bulk_init(); + /* Initialize access handlers */ if (!CLG_(clo).simulate_cache) { CLG_(cachesim).log_1I0D = 0; diff --git a/callgrind/tests/Makefile.am b/callgrind/tests/Makefile.am index 0b7f9965db..105c3eb15a 100644 --- a/callgrind/tests/Makefile.am +++ b/callgrind/tests/Makefile.am @@ -44,9 +44,12 @@ EXTRA_DIST = \ find-source.vgtest find-source.stderr.exp find-source.post.exp \ inline-samefile.vgtest inline-samefile.stderr.exp inline-samefile.stdout.exp inline-samefile.post.exp \ inline-crossfile.vgtest inline-crossfile.stderr.exp inline-crossfile.stdout.exp inline-crossfile.post.exp \ - inline-crossfile-helper1.h inline-crossfile-helper2.h filter_inline + inline-crossfile-helper1.h inline-crossfile-helper2.h filter_inline \ + rep_bulk.c rep_bulk.vgtest rep_bulk_legacy.vgtest \ + rep_bulk.post.exp rep_bulk_legacy.post.exp rep_bulk_check.pl \ + rep_bulk.stderr.exp rep_bulk_legacy.stderr.exp -check_PROGRAMS = clreq register_desc register_desc_threads find_debuginfo simwork threads inline-samefile inline-crossfile runtime_obj_skip_c runtime_obj_skip_underflow deterministic_rng +check_PROGRAMS = clreq register_desc register_desc_threads find_debuginfo simwork threads inline-samefile inline-crossfile runtime_obj_skip_c runtime_obj_skip_underflow deterministic_rng rep_bulk if HAVE_PR_GET_AUXV check_PROGRAMS += prctl_get_auxv endif @@ -58,6 +61,10 @@ AM_CXXFLAGS += $(AM_FLAG_M3264_PRI) inline_samefile_CFLAGS = $(AM_CFLAGS) -O2 -g inline_crossfile_CFLAGS = $(AM_CFLAGS) -O2 -g +# Keep the measured part free of avoidable stack traffic: inline asm cases +# with the pointers already in registers. +rep_bulk_CFLAGS = $(AM_CFLAGS) -O2 + threads_LDADD = -lpthread deterministic_rng_LDADD = -lpthread register_desc_threads_LDADD = -lpthread diff --git a/callgrind/tests/rep_bulk.c b/callgrind/tests/rep_bulk.c new file mode 100644 index 0000000000..984fe3fc6b --- /dev/null +++ b/callgrind/tests/rep_bulk.c @@ -0,0 +1,138 @@ +/*--------------------------------------------------------------------*/ +/*--- Regression test for the CodSpeed REP bulk cost model. ---*/ +/*--------------------------------------------------------------------*/ + +/* Each case is one callgrind part: ZERO_STATS, one rep instruction, + * DUMP_STATS. The post-processing script checks the emitted counters and + * that no new event names appear, with the cache geometry pinned by the + * .vgtest file (64 byte lines). */ + + +#if defined(__x86_64__) + +#include "../callgrind.h" + +#define COPY_SZ 10240u /* 160 x 64B lines */ +#define BACK_SZ 4096u +#define STOS_SZ 4096u +#define FLUSH_SZ (256u * 1024u) + +static unsigned char srcbuf[COPY_SZ + 128] __attribute__((aligned(64))); +static unsigned char dstbuf[COPY_SZ + 128] __attribute__((aligned(64))); +static volatile unsigned char flushbuf[FLUSH_SZ] __attribute__((aligned(64))); + +/* Evict the source/destination lines from D1. */ +static void flush_d1(void) +{ + unsigned i; + for (i = 0; i < FLUSH_SZ; i += 64) + flushbuf[i] = (unsigned char)i; +} + +static int verify_copy(void) +{ + unsigned i; + for (i = 0; i < COPY_SZ; i++) + if (dstbuf[i] != srcbuf[i]) + return 0; + return 1; +} + +/* Read every line of both copy buffers, so they are the most recently used + * lines in D1. The compiler may lower buffer initialization to a libc call + * whose first use runs lazy symbol binding in ld.so; those loads can evict + * buffer lines when the copy buffers already fill most ways of a set. Every + * load feeds the sink: Valgrind drops loads whose result is unused. */ +static volatile unsigned warm_sink; + +__attribute__((noinline)) +static void warm_buffers(void) +{ + unsigned i, acc = 0; + for (i = 0; i < COPY_SZ; i += 64) + acc += srcbuf[i] + dstbuf[i]; + warm_sink = acc; +} + +__attribute__((noinline)) +static void measure_copy(unsigned char *d, const unsigned char *s, unsigned long n) +{ + CALLGRIND_ZERO_STATS; + __asm__ __volatile__("rep movsb" + : "+D"(d), "+S"(s), "+c"(n) + : "a"(0u) + : "memory"); + CALLGRIND_DUMP_STATS; +} + +int main(void) +{ + unsigned char *d; + const unsigned char *s; + unsigned long n; + unsigned i; + + for (i = 0; i < COPY_SZ; i++) { + srcbuf[i] = (unsigned char)(i * 7u); + dstbuf[i] = 0; + } + /* Make both copy buffers D1-resident before the first measured part so + * the "hot" cases really have no misses. */ + warm_buffers(); + + /* part 0: one-byte forward copy (single element) */ + measure_copy(dstbuf, srcbuf, 1); + + /* part 1: hot 10 KiB forward copy, both buffers 64B-aligned */ + measure_copy(dstbuf, srcbuf, COPY_SZ); + if (!verify_copy()) + return 2; + + /* part 2: zero-length rep (no element may be charged) */ + measure_copy(dstbuf, srcbuf, 0); + if (!verify_copy()) + return 4; + + /* part 3: backward overlapping move (memmove pattern) */ + for (i = 0; i < BACK_SZ; i++) + dstbuf[i] = (unsigned char)(i * 13u + 1u); + d = dstbuf + 8 + BACK_SZ - 1; s = dstbuf + BACK_SZ - 1; n = BACK_SZ; + CALLGRIND_ZERO_STATS; + __asm__ __volatile__("std\n\trep movsb\n\tcld" + : "+D"(d), "+S"(s), "+c"(n) + : "a"(0u) + : "memory", "cc"); + CALLGRIND_DUMP_STATS; + for (i = 0; i < BACK_SZ; i++) + if (dstbuf[i + 8] != (unsigned char)(i * 13u + 1u)) + return 5; + + /* part 4: hot rep stosb */ + d = dstbuf; s = srcbuf; n = STOS_SZ; + CALLGRIND_ZERO_STATS; + __asm__ __volatile__("rep stosb" + : "+D"(d), "+S"(s), "+c"(n) + : "a"((unsigned char)0x5a) + : "memory"); + CALLGRIND_DUMP_STATS; + for (i = 0; i < STOS_SZ; i++) + if (dstbuf[i] != 0x5a) + return 6; + + /* part 5: cold 10 KiB forward copy after flushing D1 */ + flush_d1(); + measure_copy(dstbuf, srcbuf, COPY_SZ); + if (!verify_copy()) + return 3; + + return 0; +} + +#else /* !__x86_64__ */ + +int main(void) +{ + return 0; +} + +#endif diff --git a/callgrind/tests/rep_bulk.post.exp b/callgrind/tests/rep_bulk.post.exp new file mode 100644 index 0000000000..884714fb0d --- /dev/null +++ b/callgrind/tests/rep_bulk.post.exp @@ -0,0 +1,9 @@ +events: ok +one_byte_bulk_cost: ok +hot_lines_suppressed: ok +instruction_refs_kept: ok +bulk_cycles: ok +backward: ok +stos: ok +ll_resident_refs_suppressed: ok +cache_independent_cycles: ok diff --git a/callgrind/tests/rep_bulk.stderr.exp b/callgrind/tests/rep_bulk.stderr.exp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/callgrind/tests/rep_bulk.vgtest b/callgrind/tests/rep_bulk.vgtest new file mode 100644 index 0000000000..0445cc0825 --- /dev/null +++ b/callgrind/tests/rep_bulk.vgtest @@ -0,0 +1,5 @@ +prereq: rm -f callgrind.rep_bulk.out* && test "$(uname -m)" = x86_64 +prog: rep_bulk +vgopts: -q --combine-dumps=yes --cache-sim=yes --cycle-estimation=yes --I1=32768,8,64 --D1=32768,8,64 --LL=8388608,16,64 --callgrind-out-file=callgrind.rep_bulk.out +post: perl ./rep_bulk_check.pl callgrind.rep_bulk.out on +cleanup: rm -f callgrind.rep_bulk.out callgrind.rep_bulk.out.* diff --git a/callgrind/tests/rep_bulk_check.pl b/callgrind/tests/rep_bulk_check.pl new file mode 100755 index 0000000000..d7f74b842c --- /dev/null +++ b/callgrind/tests/rep_bulk_check.pl @@ -0,0 +1,73 @@ +#!/usr/bin/perl +# Check the callgrind output of the rep_bulk test. +# Usage: rep_bulk_check.pl on|off + +use strict; +use warnings; + +my ($file, $mode) = @ARGV; +open(my $fh, '<', $file) or die "cannot open $file: $!\n"; +my (@events, @parts); +while (<$fh>) { + @events = split(' ', $1) if !@events && /^events:\s+(.*)$/; + if (/^summary:\s+(.*)$/) { + my %p; + @p{@events} = split(' ', $1); + push @parts, \%p; + } +} +close($fh); +die "expected at least 6 dump parts, got " . @parts . "\n" if @parts < 6; + +sub val { + my ($part, $name) = @_; + return $parts[$part]{$name} // die "part $part has no event $name\n"; +} +# Deltas of a part against part 2 (zero-length rep: pure wrapper cost). +sub delta { my ($part, $name) = @_; return val($part, $name) - val(2, $name); } +# Cold 10 KiB copy: one miss per 64B line (160), well below one per element. +sub cold_misses { return !grep { delta(5, $_) < 160 || delta(5, $_) >= 1024 } @_; } + +my $fail = 0; +sub check { + my ($name, $ok) = @_; + print "$name: ", ($ok ? "ok" : "MISMATCH"), "\n"; + $fail = 1 unless $ok; +} + +check('events', "@events" eq 'Ir Dr Dw I1mr D1mr D1mw ILmr DLmr DLmw Ct Cl'); +if ($mode eq 'on') { + check('one_byte_bulk_cost', val(0, 'Ct') > val(2, 'Ct')); + check('hot_lines_suppressed', + !grep { delta(1, $_) != 0 } qw(Dr Dw D1mr D1mw)); + # Instruction-side counters are never modeled: one element executes one + # more IMark than the zero-length dispatch (1 vs 2 dispatches). + check('instruction_refs_kept', + delta(0, 'Ir') == 1 && delta(1, 'Ir') == 10240); + # The aligned curve prices 10 KiB at about 19K centicycles, the + # misaligned one at about 38K: the class must come from the first element. + my $ct = delta(1, 'Ct'); + check('bulk_cycles', $ct > 0 && $ct < 25000 && delta(1, 'Cl') > 0); + # A 4 KiB backward copy costs more than the 10 KiB forward one only if + # every element keeps the backward curve. + check('backward', + delta(3, 'Ct') > delta(1, 'Ct') + && delta(3, 'Dr') == 0 && delta(3, 'Dw') == 0); + check('stos', + val(4, 'Ct') > val(2, 'Ct') + && delta(4, 'Dr') == 0 && delta(4, 'Dw') == 0); + # L1-cold but LL-resident lines stream at near-hit cost: no refs. + check('ll_resident_refs_suppressed', + !grep { delta(5, $_) != 0 } qw(Dr Dw D1mr D1mw)); + check('cache_independent_cycles', + val(1, 'Ct') == val(5, 'Ct') && val(1, 'Cl') == val(5, 'Cl')); +} else { + check('element_refs', delta(0, 'Dr') == 1 && delta(0, 'Dw') == 1); + check('legacy_hot_refs', delta(1, 'Dr') == 10240 && delta(1, 'Dw') == 10240); + check('legacy_cycles', delta(1, 'Ct') > 500000); + check('legacy_zero_dispatch', delta(0, 'Ct') >= 100); + check('legacy_backward_refs', delta(3, 'Dr') == 4096 && delta(3, 'Dw') == 4096); + check('legacy_stos_refs', delta(4, 'Dr') == 0 && delta(4, 'Dw') == 4096); + check('legacy_cold_refs', delta(5, 'Dr') >= 10240 && cold_misses(qw(D1mr D1mw))); +} +exit $fail; diff --git a/callgrind/tests/rep_bulk_legacy.post.exp b/callgrind/tests/rep_bulk_legacy.post.exp new file mode 100644 index 0000000000..192e7303e5 --- /dev/null +++ b/callgrind/tests/rep_bulk_legacy.post.exp @@ -0,0 +1,8 @@ +events: ok +element_refs: ok +legacy_hot_refs: ok +legacy_cycles: ok +legacy_zero_dispatch: ok +legacy_backward_refs: ok +legacy_stos_refs: ok +legacy_cold_refs: ok diff --git a/callgrind/tests/rep_bulk_legacy.stderr.exp b/callgrind/tests/rep_bulk_legacy.stderr.exp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/callgrind/tests/rep_bulk_legacy.vgtest b/callgrind/tests/rep_bulk_legacy.vgtest new file mode 100644 index 0000000000..b052754ca8 --- /dev/null +++ b/callgrind/tests/rep_bulk_legacy.vgtest @@ -0,0 +1,5 @@ +prereq: rm -f callgrind.rep_bulk.out* && test "$(uname -m)" = x86_64 +prog: rep_bulk +vgopts: -q --combine-dumps=yes --cache-sim=yes --cycle-estimation=yes --rep-bulk-cost=no --I1=32768,8,64 --D1=32768,8,64 --LL=8388608,16,64 --callgrind-out-file=callgrind.rep_bulk.out +post: perl ./rep_bulk_check.pl callgrind.rep_bulk.out off +cleanup: rm -f callgrind.rep_bulk.out callgrind.rep_bulk.out.* diff --git a/callgrind/threads.c b/callgrind/threads.c index 642813e4cb..8f741afbc7 100644 --- a/callgrind/threads.c +++ b/callgrind/threads.c @@ -441,6 +441,9 @@ void CLG_(init_exec_state)(exec_state* es) es->jmps_passed = 0; es->bbcc = 0; es->nonskipped = 0; + es->rep_next_dst = 0; + es->rep_next_rem = 0; + es->rep_align = 0; } @@ -540,6 +543,9 @@ exec_state* exec_state_save(void) es->jmps_passed = CLG_(current_state).jmps_passed; es->bbcc = CLG_(current_state).bbcc; es->nonskipped = CLG_(current_state).nonskipped; + es->rep_next_dst = CLG_(current_state).rep_next_dst; + es->rep_next_rem = CLG_(current_state).rep_next_rem; + es->rep_align = CLG_(current_state).rep_align; CLG_ASSERT(es->cost == CLG_(current_state).cost); CLG_DEBUGIF(1) { @@ -565,6 +571,9 @@ exec_state* exec_state_restore(void) CLG_(current_state).jmps_passed = es->jmps_passed; CLG_(current_state).bbcc = es->bbcc; CLG_(current_state).nonskipped = es->nonskipped; + CLG_(current_state).rep_next_dst = es->rep_next_dst; + CLG_(current_state).rep_next_rem = es->rep_next_rem; + CLG_(current_state).rep_align = es->rep_align; CLG_(current_state).cost = es->cost; CLG_(current_state).sig = es->sig;