From b9db6251abcd3708087a18a73d3568a44d51b5ee Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Tue, 22 Sep 2026 12:30:35 +0000 Subject: [PATCH] rsync 3.5.1: read the protocol 33 block stat, and stop truncating byte totals rsync 3.5.1 bumped the protocol to 33 and added a line to --stats: Number of 4 KiB logical blocks touched: 857 printed between "Literal data" and "Matched data", and only when both ends negotiate 33. Nothing gated 3.5.1 out before this -- every capability check is a lower bound and the parser drops lines it does not recognise -- but the new number fell through to the log instead of being read. Parse it into RsyncStats.logicalBlocksTouched, gate it on a new blockStats capability taken from the negotiated protocol rather than the version string, and intersect it like the rest so a mixed 3.5/3.4 pair reports null instead of a number only one end can produce. The CLI prints the line when it is there and stays silent when it is not. Verifying that against a real 3.5.1 build turned up a separate bug. rsync prints byte totals with human_num, so under --human-readable -- which DiskPush passes by default -- "Total bytes sent: 3.50M" was being read by a [\d,]+ pattern as 3. Literal data, matched data, both byte totals and speedup were all wrong on every version, not just 3.5.1. Byte fields now read the unit suffix back through the same conversion the progress line already used; counts keep their exact comma_num parsing. Checked against real binaries on both sides: stock 3.4.1 and a 3.5.1 built from source. The full live suite passes against each, including the resume test that exercises the partial-dir validation 3.5.1 tightened. Co-Authored-By: Claude Opus 5 (1M context) --- apps/cli/src/commands/transfer.ts | 10 ++++++ docs/rsync-options.md | 6 ++++ packages/rsync-core/src/parse.test.ts | 43 +++++++++++++++++++++++ packages/rsync-core/src/parse.ts | 38 ++++++++++++++++---- packages/rsync-core/src/version.test.ts | 46 +++++++++++++++++++++++++ packages/rsync-core/src/version.ts | 13 ++++++- packages/schemas/src/events.ts | 6 ++++ tests/live/rsync-behaviour.test.ts | 31 +++++++++++++++++ 8 files changed, 185 insertions(+), 8 deletions(-) diff --git a/apps/cli/src/commands/transfer.ts b/apps/cli/src/commands/transfer.ts index 8dddb89..98d9ded 100644 --- a/apps/cli/src/commands/transfer.ts +++ b/apps/cli/src/commands/transfer.ts @@ -262,6 +262,7 @@ export async function runTransfer( // aggregate percentage alone tells you it is alive but not what it is doing. let currentFile = '' let lastProgress: import('@diskpush/schemas').RsyncProgress | null = null + let stats: import('@diskpush/schemas').RsyncStats | null = null let exitCode: number = EXIT.internal let message = '' let resumable = false @@ -291,6 +292,9 @@ export async function runTransfer( output.status(currentFile === '' ? head : `${head} ${truncatePath(currentFile, head.length)}`) break } + case 'stats': + stats = event.stats + break case 'stderr': stderr.push(event.line) break @@ -332,6 +336,11 @@ export async function runTransfer( if (lastProgress) { output.line(`Transferred: ${formatBytes(lastProgress.bytesTransferred)} in ${formatDuration(lastProgress.elapsedSeconds)}`) } + // Only rsync 3.5.0+ on both ends reports this, so the line is absent rather + // than zero everywhere else. + if (stats?.logicalBlocksTouched != null) { + output.line(`Blocks touched: ${stats.logicalBlocksTouched.toLocaleString('en-US')} x 4 KiB`) + } output.line(message) if (output.isJson) { @@ -343,6 +352,7 @@ export async function runTransfer( resumable, message, changes: summary, + stats, command: plan.display, control: plan.controlDisplay ?? null, direct: plan.direct, diff --git a/docs/rsync-options.md b/docs/rsync-options.md index 874d5d0..3d47901 100644 --- a/docs/rsync-options.md +++ b/docs/rsync-options.md @@ -74,6 +74,12 @@ the run. DiskPush parses `rsync --version` on both ends and intersects them. | `--protect-args` | 3.0.0+ | | `--secluded-args` by default | 3.2.4+ | | `--acls`, `--xattrs` | a build with support, on both ends | +| 4 KiB blocks touched in `--stats` | 3.5.0+ on both ends (protocol 33) | + +DiskPush runs against every rsync from 2.6.9 up, including 3.5.1. A newer rsync +is never gated out: the table above is all lower bounds, and an unrecognised +`--stats` line from a future release is passed through to the log rather than +treated as an error. The remote side is only known once `diskpush connections test` has run; its report is cached against the connection. Without it, DiskPush uses the local diff --git a/packages/rsync-core/src/parse.test.ts b/packages/rsync-core/src/parse.test.ts index f6ef2d9..6861d75 100644 --- a/packages/rsync-core/src/parse.test.ts +++ b/packages/rsync-core/src/parse.test.ts @@ -126,6 +126,49 @@ describe('parseStatsLine', () => { }) }) + it('reads human-readable byte totals, which is how DiskPush runs rsync', () => { + const stats = emptyStats() + // Verbatim from `rsync --human-readable --stats`; reading these as plain + // digits truncated "3.50M" to 3. + for (const line of [ + 'Literal data: 3.50M bytes', + 'Matched data: 0 bytes', + 'Total bytes sent: 3.50M', + 'Total bytes received: 85', + 'total size is 3.50M speedup is 1.00', + ]) { + expect(parseStatsLine(line, stats)).toBe(true) + } + expect(stats).toMatchObject({ + literalBytes: 3_500_000, + matchedBytes: 0, + totalBytesSent: 3_500_000, + totalBytesReceived: 85, + speedup: 1, + }) + }) + + it('reads the protocol 33 block stat that rsync 3.5 added', () => { + const stats = emptyStats() + // rsync prints this between "Literal data" and "Matched data". + for (const line of [ + 'Literal data: 1,234,567 bytes', + 'Number of 4 KiB logical blocks touched: 12,345', + 'Matched data: 89 bytes', + ]) { + expect(parseStatsLine(line, stats)).toBe(true) + } + expect(stats.logicalBlocksTouched).toBe(12345) + expect(stats.literalBytes).toBe(1234567) + expect(stats.matchedBytes).toBe(89) + }) + + it('leaves the block stat null on an older rsync that never prints it', () => { + const stats = emptyStats() + expect(parseStatsLine('Literal data: 1,234,567 bytes', stats)).toBe(true) + expect(stats.logicalBlocksTouched).toBeNull() + }) + it('ignores lines that are not stats', () => { expect(parseStatsLine('>f+++++++++ a.txt', emptyStats())).toBe(false) }) diff --git a/packages/rsync-core/src/parse.ts b/packages/rsync-core/src/parse.ts index e88ff1f..7f7750a 100644 --- a/packages/rsync-core/src/parse.ts +++ b/packages/rsync-core/src/parse.ts @@ -100,13 +100,29 @@ export function classifyItemize(itemize: string): ChangeAction { return 'update' } -const STATS_PATTERNS: Array<[keyof RsyncStats, RegExp]> = [ +/** + * Counts. rsync prints these with `comma_num`, so they are exact integers + * whatever `--human-readable` is doing. + */ +const COUNT_PATTERNS: Array<[keyof RsyncStats, RegExp]> = [ ['filesTotal', /^Number of files:\s+([\d,]+)/], ['filesTransferred', /^Number of (?:regular files transferred|files transferred):\s+([\d,]+)/], - ['literalBytes', /^Literal data:\s+([\d,]+)/], - ['matchedBytes', /^Matched data:\s+([\d,]+)/], - ['totalBytesSent', /^Total bytes sent:\s+([\d,]+)/], - ['totalBytesReceived', /^Total bytes received:\s+([\d,]+)/], + // Protocol 33 (rsync 3.5.0+) prints this between "Literal data" and + // "Matched data". Older rsync omits the line entirely. + ['logicalBlocksTouched', /^Number of 4 KiB logical blocks touched:\s+([\d,]+)/], +] + +/** + * Byte totals. rsync prints these with `human_num`, so under + * `--human-readable` -- which DiskPush passes by default -- they arrive as + * "3.50M" rather than "3,500,000". The unit suffix has to be read back or the + * value truncates to its leading digit. + */ +const BYTE_PATTERNS: Array<[keyof RsyncStats, RegExp]> = [ + ['literalBytes', /^Literal data:\s+([\d,.]+)([KMGTP])?/], + ['matchedBytes', /^Matched data:\s+([\d,.]+)([KMGTP])?/], + ['totalBytesSent', /^Total bytes sent:\s+([\d,.]+)([KMGTP])?/], + ['totalBytesReceived', /^Total bytes received:\s+([\d,.]+)([KMGTP])?/], ] export function emptyStats(): RsyncStats { @@ -116,6 +132,7 @@ export function emptyStats(): RsyncStats { totalBytesSent: null, totalBytesReceived: null, literalBytes: null, + logicalBlocksTouched: null, matchedBytes: null, speedup: null, } @@ -123,14 +140,21 @@ export function emptyStats(): RsyncStats { /** Folds one `--stats` line into an accumulating stats object. */ export function parseStatsLine(line: string, into: RsyncStats): boolean { - for (const [key, pattern] of STATS_PATTERNS) { + for (const [key, pattern] of COUNT_PATTERNS) { const match = pattern.exec(line) if (match) { into[key] = Number((match[1] ?? '').replaceAll(',', '')) return true } } - const speedup = /^(?:total size is [\d,]+\s+)?speedup is ([\d.]+)/i.exec(line) + for (const [key, pattern] of BYTE_PATTERNS) { + const match = pattern.exec(line) + if (match) { + into[key] = toNumber(match[1] ?? '0', match[2]) + return true + } + } + const speedup = /^(?:total size is [\d,.]+[KMGTP]?\s+)?speedup is ([\d.]+)/i.exec(line) if (speedup) { into.speedup = Number(speedup[1]) return true diff --git a/packages/rsync-core/src/version.test.ts b/packages/rsync-core/src/version.test.ts index 81c36b1..3baf419 100644 --- a/packages/rsync-core/src/version.test.ts +++ b/packages/rsync-core/src/version.test.ts @@ -14,6 +14,20 @@ Checksum list: Compress list: zstd lz4 zlibx zlib none` +const RSYNC_351 = `rsync version 3.5.1 protocol version 33 +Copyright (C) 1996-2026 by Andrew Tridgell, Wayne Davison, and others. +Capabilities: + 64-bit files, 64-bit inums, 64-bit timestamps, 64-bit long ints, + socketpairs, symlinks, symtimes, hardlinks, hardlink-specials, + hardlink-symlinks, IPv6, atimes, batchfiles, inplace, append, ACLs, + xattrs, optional secluded-args, iconv, prealloc, stop-at, no crtimes +Optimizations: + SIMD-roll, no asm-roll, openssl-crypto, no asm-MD5 +Checksum list: + xxh128 xxh3 xxh64 md5 md4 sha1 none +Compress list: + zstd lz4 zlibx zlib none` + const RSYNC_316 = `rsync version 3.1.6 protocol version 31 Capabilities: 64-bit files, socketpairs, hardlinks, symlinks, ACLs, xattrs, protect-args @@ -29,6 +43,10 @@ describe('parseRsyncVersion', () => { expect(parseRsyncVersion(RSYNC_341)).toMatchObject({ major: 3, minor: 4, patch: 1 }) }) + it('reads 3.5.1, the current release', () => { + expect(parseRsyncVersion(RSYNC_351)).toMatchObject({ major: 3, minor: 5, patch: 1, raw: '3.5.1' }) + }) + it('defaults a missing patch to zero', () => { expect(parseRsyncVersion('rsync version 3.2 protocol version 31')).toMatchObject({ major: 3, minor: 2, patch: 0 }) }) @@ -67,6 +85,27 @@ describe('parseRsyncCapabilities', () => { expect(parseRsyncCapabilities(RSYNC_341).mkpath).toBe(true) expect(parseRsyncCapabilities(RSYNC_316).mkpath).toBe(false) }) + + it('reads protocol 33 from 3.5.1 and keeps every older capability', () => { + const caps = parseRsyncCapabilities(RSYNC_351) + expect(caps.protocol).toBe(33) + expect(caps.secludedArgsByDefault).toBe(true) + expect(caps.mkpath).toBe(true) + expect(caps.zstd).toBe(true) + expect(caps.acls).toBe(true) + expect(caps.xattrs).toBe(true) + }) + + it('gates the 4 KiB block stat on protocol 33', () => { + expect(parseRsyncCapabilities(RSYNC_351).blockStats).toBe(true) + expect(parseRsyncCapabilities(RSYNC_341).blockStats).toBe(false) + expect(parseRsyncCapabilities(RSYNC_316).blockStats).toBe(false) + }) + + it('falls back to the version when the banner prints no protocol line', () => { + expect(parseRsyncCapabilities('rsync version 3.5.1').blockStats).toBe(true) + expect(parseRsyncCapabilities('rsync version 3.4.1').blockStats).toBe(false) + }) }) describe('atLeast', () => { @@ -82,6 +121,13 @@ describe('atLeast', () => { }) describe('intersectCapabilities', () => { + it('drops the block stat unless both ends speak protocol 33', () => { + const modern = parseRsyncCapabilities(RSYNC_351) + expect(intersectCapabilities(modern, parseRsyncCapabilities(RSYNC_341)).blockStats).toBe(false) + expect(intersectCapabilities(modern, modern).blockStats).toBe(true) + expect(intersectCapabilities(modern, parseRsyncCapabilities(RSYNC_341)).protocol).toBe(32) + }) + it('takes the weaker of the two ends', () => { const merged = intersectCapabilities(parseRsyncCapabilities(RSYNC_341), parseRsyncCapabilities(RSYNC_316)) expect(merged.zstd).toBe(false) diff --git a/packages/rsync-core/src/version.ts b/packages/rsync-core/src/version.ts index 8eec1a6..512a78a 100644 --- a/packages/rsync-core/src/version.ts +++ b/packages/rsync-core/src/version.ts @@ -18,6 +18,11 @@ export type RsyncCapabilities = { secludedArgsAvailable: boolean /** `--mkpath` (3.2.3+). */ mkpath: boolean + /** + * `--stats` reports the number of 4 KiB logical blocks touched. This rides on + * protocol 33, which rsync 3.5.0 introduced, and both ends must negotiate it. + */ + blockStats: boolean acls: boolean xattrs: boolean hardLinks: boolean @@ -30,6 +35,7 @@ const UNKNOWN: RsyncCapabilities = { secludedArgsByDefault: false, secludedArgsAvailable: false, mkpath: false, + blockStats: false, acls: false, xattrs: false, hardLinks: true, @@ -68,6 +74,7 @@ export function parseRsyncCapabilities(banner: string): RsyncCapabilities { const lower = banner.toLowerCase() const protocolMatch = /protocol\s+version\s+(\d+)/i.exec(banner) + const protocol = protocolMatch ? Number(protocolMatch[1]) : null // The compress list is authoritative for zstd; the version only makes it possible. const compressList = /compress list:\s*([^\n]*)/i.exec(lower) @@ -75,11 +82,14 @@ export function parseRsyncCapabilities(banner: string): RsyncCapabilities { return { version, - protocol: protocolMatch ? Number(protocolMatch[1]) : null, + protocol, zstd, secludedArgsByDefault: atLeast(version, 3, 2, 4), secludedArgsAvailable: atLeast(version, 3, 0, 0), mkpath: atLeast(version, 3, 2, 3), + // The negotiated protocol is what actually decides this; the version is + // only a fallback for a banner that does not print a protocol line. + blockStats: protocol === null ? atLeast(version, 3, 5, 0) : protocol >= 33, acls: /\bACLs\b/i.test(banner) && !/no ACLs/i.test(banner), xattrs: /\bxattrs\b/i.test(banner) && !/no xattrs/i.test(banner), hardLinks: !/no hardlinks/i.test(lower), @@ -105,6 +115,7 @@ export function intersectCapabilities(a: RsyncCapabilities, b: RsyncCapabilities secludedArgsByDefault: a.secludedArgsByDefault && b.secludedArgsByDefault, secludedArgsAvailable: a.secludedArgsAvailable && b.secludedArgsAvailable, mkpath: a.mkpath && b.mkpath, + blockStats: a.blockStats && b.blockStats, acls: a.acls && b.acls, xattrs: a.xattrs && b.xattrs, hardLinks: a.hardLinks && b.hardLinks, diff --git a/packages/schemas/src/events.ts b/packages/schemas/src/events.ts index e75520e..71e2ed5 100644 --- a/packages/schemas/src/events.ts +++ b/packages/schemas/src/events.ts @@ -18,6 +18,12 @@ export type RsyncStats = { totalBytesSent: number | null totalBytesReceived: number | null literalBytes: number | null + /** + * Distinct 4 KiB logical file blocks the receiver touched. rsync prints this + * only when both ends negotiate protocol 33 (rsync 3.5.0+), so it stays null + * against anything older rather than being reported as zero. + */ + logicalBlocksTouched: number | null matchedBytes: number | null speedup: number | null } diff --git a/tests/live/rsync-behaviour.test.ts b/tests/live/rsync-behaviour.test.ts index 3f8f622..f0d2f8a 100644 --- a/tests/live/rsync-behaviour.test.ts +++ b/tests/live/rsync-behaviour.test.ts @@ -224,3 +224,34 @@ describe.skipIf(!hasRsync)('live rsync: resume', () => { expect(readFileSync(join(dst, 'big.bin')).equals(big)).toBe(true) }, 60_000) }) + +describe.skipIf(!hasRsync)('live rsync: --stats against the installed binary', () => { + it('reads byte totals back at full size rather than truncating them', async () => { + const { src, dst } = tree('stats-bytes') + // Large enough that rsync renders the totals as "3.50M" under + // --human-readable, which DiskPush passes by default. + writeFileSync(join(src, 'blob.bin'), Buffer.alloc(3_500_000, 7)) + + const result = await sync(src, dst) + expect(result.ok).toBe(true) + expect(result.stats).not.toBeNull() + expect(result.stats?.totalBytesSent).toBeGreaterThan(3_000_000) + expect(result.stats?.literalBytes).toBeGreaterThan(3_000_000) + expect(result.stats?.speedup).toBeGreaterThan(0) + }, 60_000) + + it('reports 4 KiB blocks touched exactly when this rsync speaks protocol 33', async () => { + const { src, dst } = tree('stats-blocks') + writeFileSync(join(src, 'blob.bin'), Buffer.alloc(200_000, 3)) + + const result = await sync(src, dst) + expect(result.ok).toBe(true) + // rsync 3.5.0+ prints the stat; anything older omits the line, and the + // field stays null rather than being reported as zero. + if (capabilities.blockStats) { + expect(result.stats?.logicalBlocksTouched).toBeGreaterThan(0) + } else { + expect(result.stats?.logicalBlocksTouched).toBeNull() + } + }, 60_000) +})