diff --git a/apps/cli/src/commands/transfer.ts b/apps/cli/src/commands/transfer.ts index 8dddb89..98d9ded 100644 --- a/apps/cli/src/commands/transfer.ts +++ b/apps/cli/src/commands/transfer.ts @@ -262,6 +262,7 @@ export async function runTransfer( // aggregate percentage alone tells you it is alive but not what it is doing. let currentFile = '' let lastProgress: import('@diskpush/schemas').RsyncProgress | null = null + let stats: import('@diskpush/schemas').RsyncStats | null = null let exitCode: number = EXIT.internal let message = '' let resumable = false @@ -291,6 +292,9 @@ export async function runTransfer( output.status(currentFile === '' ? head : `${head} ${truncatePath(currentFile, head.length)}`) break } + case 'stats': + stats = event.stats + break case 'stderr': stderr.push(event.line) break @@ -332,6 +336,11 @@ export async function runTransfer( if (lastProgress) { output.line(`Transferred: ${formatBytes(lastProgress.bytesTransferred)} in ${formatDuration(lastProgress.elapsedSeconds)}`) } + // Only rsync 3.5.0+ on both ends reports this, so the line is absent rather + // than zero everywhere else. + if (stats?.logicalBlocksTouched != null) { + output.line(`Blocks touched: ${stats.logicalBlocksTouched.toLocaleString('en-US')} x 4 KiB`) + } output.line(message) if (output.isJson) { @@ -343,6 +352,7 @@ export async function runTransfer( resumable, message, changes: summary, + stats, command: plan.display, control: plan.controlDisplay ?? null, direct: plan.direct, diff --git a/docs/rsync-options.md b/docs/rsync-options.md index 874d5d0..3d47901 100644 --- a/docs/rsync-options.md +++ b/docs/rsync-options.md @@ -74,6 +74,12 @@ the run. DiskPush parses `rsync --version` on both ends and intersects them. | `--protect-args` | 3.0.0+ | | `--secluded-args` by default | 3.2.4+ | | `--acls`, `--xattrs` | a build with support, on both ends | +| 4 KiB blocks touched in `--stats` | 3.5.0+ on both ends (protocol 33) | + +DiskPush runs against every rsync from 2.6.9 up, including 3.5.1. A newer rsync +is never gated out: the table above is all lower bounds, and an unrecognised +`--stats` line from a future release is passed through to the log rather than +treated as an error. The remote side is only known once `diskpush connections test` has run; its report is cached against the connection. Without it, DiskPush uses the local diff --git a/packages/rsync-core/src/parse.test.ts b/packages/rsync-core/src/parse.test.ts index f6ef2d9..6861d75 100644 --- a/packages/rsync-core/src/parse.test.ts +++ b/packages/rsync-core/src/parse.test.ts @@ -126,6 +126,49 @@ describe('parseStatsLine', () => { }) }) + it('reads human-readable byte totals, which is how DiskPush runs rsync', () => { + const stats = emptyStats() + // Verbatim from `rsync --human-readable --stats`; reading these as plain + // digits truncated "3.50M" to 3. + for (const line of [ + 'Literal data: 3.50M bytes', + 'Matched data: 0 bytes', + 'Total bytes sent: 3.50M', + 'Total bytes received: 85', + 'total size is 3.50M speedup is 1.00', + ]) { + expect(parseStatsLine(line, stats)).toBe(true) + } + expect(stats).toMatchObject({ + literalBytes: 3_500_000, + matchedBytes: 0, + totalBytesSent: 3_500_000, + totalBytesReceived: 85, + speedup: 1, + }) + }) + + it('reads the protocol 33 block stat that rsync 3.5 added', () => { + const stats = emptyStats() + // rsync prints this between "Literal data" and "Matched data". + for (const line of [ + 'Literal data: 1,234,567 bytes', + 'Number of 4 KiB logical blocks touched: 12,345', + 'Matched data: 89 bytes', + ]) { + expect(parseStatsLine(line, stats)).toBe(true) + } + expect(stats.logicalBlocksTouched).toBe(12345) + expect(stats.literalBytes).toBe(1234567) + expect(stats.matchedBytes).toBe(89) + }) + + it('leaves the block stat null on an older rsync that never prints it', () => { + const stats = emptyStats() + expect(parseStatsLine('Literal data: 1,234,567 bytes', stats)).toBe(true) + expect(stats.logicalBlocksTouched).toBeNull() + }) + it('ignores lines that are not stats', () => { expect(parseStatsLine('>f+++++++++ a.txt', emptyStats())).toBe(false) }) diff --git a/packages/rsync-core/src/parse.ts b/packages/rsync-core/src/parse.ts index e88ff1f..7f7750a 100644 --- a/packages/rsync-core/src/parse.ts +++ b/packages/rsync-core/src/parse.ts @@ -100,13 +100,29 @@ export function classifyItemize(itemize: string): ChangeAction { return 'update' } -const STATS_PATTERNS: Array<[keyof RsyncStats, RegExp]> = [ +/** + * Counts. rsync prints these with `comma_num`, so they are exact integers + * whatever `--human-readable` is doing. + */ +const COUNT_PATTERNS: Array<[keyof RsyncStats, RegExp]> = [ ['filesTotal', /^Number of files:\s+([\d,]+)/], ['filesTransferred', /^Number of (?:regular files transferred|files transferred):\s+([\d,]+)/], - ['literalBytes', /^Literal data:\s+([\d,]+)/], - ['matchedBytes', /^Matched data:\s+([\d,]+)/], - ['totalBytesSent', /^Total bytes sent:\s+([\d,]+)/], - ['totalBytesReceived', /^Total bytes received:\s+([\d,]+)/], + // Protocol 33 (rsync 3.5.0+) prints this between "Literal data" and + // "Matched data". Older rsync omits the line entirely. + ['logicalBlocksTouched', /^Number of 4 KiB logical blocks touched:\s+([\d,]+)/], +] + +/** + * Byte totals. rsync prints these with `human_num`, so under + * `--human-readable` -- which DiskPush passes by default -- they arrive as + * "3.50M" rather than "3,500,000". The unit suffix has to be read back or the + * value truncates to its leading digit. + */ +const BYTE_PATTERNS: Array<[keyof RsyncStats, RegExp]> = [ + ['literalBytes', /^Literal data:\s+([\d,.]+)([KMGTP])?/], + ['matchedBytes', /^Matched data:\s+([\d,.]+)([KMGTP])?/], + ['totalBytesSent', /^Total bytes sent:\s+([\d,.]+)([KMGTP])?/], + ['totalBytesReceived', /^Total bytes received:\s+([\d,.]+)([KMGTP])?/], ] export function emptyStats(): RsyncStats { @@ -116,6 +132,7 @@ export function emptyStats(): RsyncStats { totalBytesSent: null, totalBytesReceived: null, literalBytes: null, + logicalBlocksTouched: null, matchedBytes: null, speedup: null, } @@ -123,14 +140,21 @@ export function emptyStats(): RsyncStats { /** Folds one `--stats` line into an accumulating stats object. */ export function parseStatsLine(line: string, into: RsyncStats): boolean { - for (const [key, pattern] of STATS_PATTERNS) { + for (const [key, pattern] of COUNT_PATTERNS) { const match = pattern.exec(line) if (match) { into[key] = Number((match[1] ?? '').replaceAll(',', '')) return true } } - const speedup = /^(?:total size is [\d,]+\s+)?speedup is ([\d.]+)/i.exec(line) + for (const [key, pattern] of BYTE_PATTERNS) { + const match = pattern.exec(line) + if (match) { + into[key] = toNumber(match[1] ?? '0', match[2]) + return true + } + } + const speedup = /^(?:total size is [\d,.]+[KMGTP]?\s+)?speedup is ([\d.]+)/i.exec(line) if (speedup) { into.speedup = Number(speedup[1]) return true diff --git a/packages/rsync-core/src/version.test.ts b/packages/rsync-core/src/version.test.ts index 81c36b1..3baf419 100644 --- a/packages/rsync-core/src/version.test.ts +++ b/packages/rsync-core/src/version.test.ts @@ -14,6 +14,20 @@ Checksum list: Compress list: zstd lz4 zlibx zlib none` +const RSYNC_351 = `rsync version 3.5.1 protocol version 33 +Copyright (C) 1996-2026 by Andrew Tridgell, Wayne Davison, and others. +Capabilities: + 64-bit files, 64-bit inums, 64-bit timestamps, 64-bit long ints, + socketpairs, symlinks, symtimes, hardlinks, hardlink-specials, + hardlink-symlinks, IPv6, atimes, batchfiles, inplace, append, ACLs, + xattrs, optional secluded-args, iconv, prealloc, stop-at, no crtimes +Optimizations: + SIMD-roll, no asm-roll, openssl-crypto, no asm-MD5 +Checksum list: + xxh128 xxh3 xxh64 md5 md4 sha1 none +Compress list: + zstd lz4 zlibx zlib none` + const RSYNC_316 = `rsync version 3.1.6 protocol version 31 Capabilities: 64-bit files, socketpairs, hardlinks, symlinks, ACLs, xattrs, protect-args @@ -29,6 +43,10 @@ describe('parseRsyncVersion', () => { expect(parseRsyncVersion(RSYNC_341)).toMatchObject({ major: 3, minor: 4, patch: 1 }) }) + it('reads 3.5.1, the current release', () => { + expect(parseRsyncVersion(RSYNC_351)).toMatchObject({ major: 3, minor: 5, patch: 1, raw: '3.5.1' }) + }) + it('defaults a missing patch to zero', () => { expect(parseRsyncVersion('rsync version 3.2 protocol version 31')).toMatchObject({ major: 3, minor: 2, patch: 0 }) }) @@ -67,6 +85,27 @@ describe('parseRsyncCapabilities', () => { expect(parseRsyncCapabilities(RSYNC_341).mkpath).toBe(true) expect(parseRsyncCapabilities(RSYNC_316).mkpath).toBe(false) }) + + it('reads protocol 33 from 3.5.1 and keeps every older capability', () => { + const caps = parseRsyncCapabilities(RSYNC_351) + expect(caps.protocol).toBe(33) + expect(caps.secludedArgsByDefault).toBe(true) + expect(caps.mkpath).toBe(true) + expect(caps.zstd).toBe(true) + expect(caps.acls).toBe(true) + expect(caps.xattrs).toBe(true) + }) + + it('gates the 4 KiB block stat on protocol 33', () => { + expect(parseRsyncCapabilities(RSYNC_351).blockStats).toBe(true) + expect(parseRsyncCapabilities(RSYNC_341).blockStats).toBe(false) + expect(parseRsyncCapabilities(RSYNC_316).blockStats).toBe(false) + }) + + it('falls back to the version when the banner prints no protocol line', () => { + expect(parseRsyncCapabilities('rsync version 3.5.1').blockStats).toBe(true) + expect(parseRsyncCapabilities('rsync version 3.4.1').blockStats).toBe(false) + }) }) describe('atLeast', () => { @@ -82,6 +121,13 @@ describe('atLeast', () => { }) describe('intersectCapabilities', () => { + it('drops the block stat unless both ends speak protocol 33', () => { + const modern = parseRsyncCapabilities(RSYNC_351) + expect(intersectCapabilities(modern, parseRsyncCapabilities(RSYNC_341)).blockStats).toBe(false) + expect(intersectCapabilities(modern, modern).blockStats).toBe(true) + expect(intersectCapabilities(modern, parseRsyncCapabilities(RSYNC_341)).protocol).toBe(32) + }) + it('takes the weaker of the two ends', () => { const merged = intersectCapabilities(parseRsyncCapabilities(RSYNC_341), parseRsyncCapabilities(RSYNC_316)) expect(merged.zstd).toBe(false) diff --git a/packages/rsync-core/src/version.ts b/packages/rsync-core/src/version.ts index 8eec1a6..512a78a 100644 --- a/packages/rsync-core/src/version.ts +++ b/packages/rsync-core/src/version.ts @@ -18,6 +18,11 @@ export type RsyncCapabilities = { secludedArgsAvailable: boolean /** `--mkpath` (3.2.3+). */ mkpath: boolean + /** + * `--stats` reports the number of 4 KiB logical blocks touched. This rides on + * protocol 33, which rsync 3.5.0 introduced, and both ends must negotiate it. + */ + blockStats: boolean acls: boolean xattrs: boolean hardLinks: boolean @@ -30,6 +35,7 @@ const UNKNOWN: RsyncCapabilities = { secludedArgsByDefault: false, secludedArgsAvailable: false, mkpath: false, + blockStats: false, acls: false, xattrs: false, hardLinks: true, @@ -68,6 +74,7 @@ export function parseRsyncCapabilities(banner: string): RsyncCapabilities { const lower = banner.toLowerCase() const protocolMatch = /protocol\s+version\s+(\d+)/i.exec(banner) + const protocol = protocolMatch ? Number(protocolMatch[1]) : null // The compress list is authoritative for zstd; the version only makes it possible. const compressList = /compress list:\s*([^\n]*)/i.exec(lower) @@ -75,11 +82,14 @@ export function parseRsyncCapabilities(banner: string): RsyncCapabilities { return { version, - protocol: protocolMatch ? Number(protocolMatch[1]) : null, + protocol, zstd, secludedArgsByDefault: atLeast(version, 3, 2, 4), secludedArgsAvailable: atLeast(version, 3, 0, 0), mkpath: atLeast(version, 3, 2, 3), + // The negotiated protocol is what actually decides this; the version is + // only a fallback for a banner that does not print a protocol line. + blockStats: protocol === null ? atLeast(version, 3, 5, 0) : protocol >= 33, acls: /\bACLs\b/i.test(banner) && !/no ACLs/i.test(banner), xattrs: /\bxattrs\b/i.test(banner) && !/no xattrs/i.test(banner), hardLinks: !/no hardlinks/i.test(lower), @@ -105,6 +115,7 @@ export function intersectCapabilities(a: RsyncCapabilities, b: RsyncCapabilities secludedArgsByDefault: a.secludedArgsByDefault && b.secludedArgsByDefault, secludedArgsAvailable: a.secludedArgsAvailable && b.secludedArgsAvailable, mkpath: a.mkpath && b.mkpath, + blockStats: a.blockStats && b.blockStats, acls: a.acls && b.acls, xattrs: a.xattrs && b.xattrs, hardLinks: a.hardLinks && b.hardLinks, diff --git a/packages/schemas/src/events.ts b/packages/schemas/src/events.ts index e75520e..71e2ed5 100644 --- a/packages/schemas/src/events.ts +++ b/packages/schemas/src/events.ts @@ -18,6 +18,12 @@ export type RsyncStats = { totalBytesSent: number | null totalBytesReceived: number | null literalBytes: number | null + /** + * Distinct 4 KiB logical file blocks the receiver touched. rsync prints this + * only when both ends negotiate protocol 33 (rsync 3.5.0+), so it stays null + * against anything older rather than being reported as zero. + */ + logicalBlocksTouched: number | null matchedBytes: number | null speedup: number | null } diff --git a/tests/live/rsync-behaviour.test.ts b/tests/live/rsync-behaviour.test.ts index 3f8f622..f0d2f8a 100644 --- a/tests/live/rsync-behaviour.test.ts +++ b/tests/live/rsync-behaviour.test.ts @@ -224,3 +224,34 @@ describe.skipIf(!hasRsync)('live rsync: resume', () => { expect(readFileSync(join(dst, 'big.bin')).equals(big)).toBe(true) }, 60_000) }) + +describe.skipIf(!hasRsync)('live rsync: --stats against the installed binary', () => { + it('reads byte totals back at full size rather than truncating them', async () => { + const { src, dst } = tree('stats-bytes') + // Large enough that rsync renders the totals as "3.50M" under + // --human-readable, which DiskPush passes by default. + writeFileSync(join(src, 'blob.bin'), Buffer.alloc(3_500_000, 7)) + + const result = await sync(src, dst) + expect(result.ok).toBe(true) + expect(result.stats).not.toBeNull() + expect(result.stats?.totalBytesSent).toBeGreaterThan(3_000_000) + expect(result.stats?.literalBytes).toBeGreaterThan(3_000_000) + expect(result.stats?.speedup).toBeGreaterThan(0) + }, 60_000) + + it('reports 4 KiB blocks touched exactly when this rsync speaks protocol 33', async () => { + const { src, dst } = tree('stats-blocks') + writeFileSync(join(src, 'blob.bin'), Buffer.alloc(200_000, 3)) + + const result = await sync(src, dst) + expect(result.ok).toBe(true) + // rsync 3.5.0+ prints the stat; anything older omits the line, and the + // field stays null rather than being reported as zero. + if (capabilities.blockStats) { + expect(result.stats?.logicalBlocksTouched).toBeGreaterThan(0) + } else { + expect(result.stats?.logicalBlocksTouched).toBeNull() + } + }, 60_000) +})