rsync 3.5.1: read the protocol 33 block stat, and stop truncating byte totals

apps/cli/src/commands/transfer.ts

@@ -262,6 +262,7 @@ export async function runTransfer(
   // aggregate percentage alone tells you it is alive but not what it is doing.
   let currentFile = ''
   let lastProgress: import('@diskpush/schemas').RsyncProgress | null = null
+  let stats: import('@diskpush/schemas').RsyncStats | null = null
   let exitCode: number = EXIT.internal
   let message = ''
   let resumable = false
@@ -291,6 +292,9 @@ export async function runTransfer(
           output.status(currentFile === '' ? head : `${head}  ${truncatePath(currentFile, head.length)}`)
           break
         }
+        case 'stats':
+          stats = event.stats
+          break
         case 'stderr':
           stderr.push(event.line)
           break
@@ -332,6 +336,11 @@ export async function runTransfer(
   if (lastProgress) {
     output.line(`Transferred: ${formatBytes(lastProgress.bytesTransferred)} in ${formatDuration(lastProgress.elapsedSeconds)}`)
   }
+  // Only rsync 3.5.0+ on both ends reports this, so the line is absent rather
+  // than zero everywhere else.
+  if (stats?.logicalBlocksTouched != null) {
+    output.line(`Blocks touched: ${stats.logicalBlocksTouched.toLocaleString('en-US')} x 4 KiB`)
+  }
   output.line(message)
 
   if (output.isJson) {
@@ -343,6 +352,7 @@ export async function runTransfer(
       resumable,
       message,
       changes: summary,
+      stats,
       command: plan.display,
       control: plan.controlDisplay ?? null,
       direct: plan.direct,

docs/rsync-options.md

@@ -74,6 +74,12 @@ the run. DiskPush parses `rsync --version` on both ends and intersects them.
 | `--protect-args` | 3.0.0+ |
 | `--secluded-args` by default | 3.2.4+ |
 | `--acls`, `--xattrs` | a build with support, on both ends |
+| 4 KiB blocks touched in `--stats` | 3.5.0+ on both ends (protocol 33) |
+
+DiskPush runs against every rsync from 2.6.9 up, including 3.5.1. A newer rsync
+is never gated out: the table above is all lower bounds, and an unrecognised
+`--stats` line from a future release is passed through to the log rather than
+treated as an error.
 
 The remote side is only known once `diskpush connections test` has run; its
 report is cached against the connection. Without it, DiskPush uses the local

packages/rsync-core/src/parse.test.ts

@@ -126,6 +126,49 @@ describe('parseStatsLine', () => {
     })
   })
 
+  it('reads human-readable byte totals, which is how DiskPush runs rsync', () => {
+    const stats = emptyStats()
+    // Verbatim from `rsync --human-readable --stats`; reading these as plain
+    // digits truncated "3.50M" to 3.
+    for (const line of [
+      'Literal data: 3.50M bytes',
+      'Matched data: 0 bytes',
+      'Total bytes sent: 3.50M',
+      'Total bytes received: 85',
+      'total size is 3.50M  speedup is 1.00',
+    ]) {
+      expect(parseStatsLine(line, stats)).toBe(true)
+    }
+    expect(stats).toMatchObject({
+      literalBytes: 3_500_000,
+      matchedBytes: 0,
+      totalBytesSent: 3_500_000,
+      totalBytesReceived: 85,
+      speedup: 1,
+    })
+  })
+
+  it('reads the protocol 33 block stat that rsync 3.5 added', () => {
+    const stats = emptyStats()
+    // rsync prints this between "Literal data" and "Matched data".
+    for (const line of [
+      'Literal data: 1,234,567 bytes',
+      'Number of 4 KiB logical blocks touched: 12,345',
+      'Matched data: 89 bytes',
+    ]) {
+      expect(parseStatsLine(line, stats)).toBe(true)
+    }
+    expect(stats.logicalBlocksTouched).toBe(12345)
+    expect(stats.literalBytes).toBe(1234567)
+    expect(stats.matchedBytes).toBe(89)
+  })
+
+  it('leaves the block stat null on an older rsync that never prints it', () => {
+    const stats = emptyStats()
+    expect(parseStatsLine('Literal data: 1,234,567 bytes', stats)).toBe(true)
+    expect(stats.logicalBlocksTouched).toBeNull()
+  })
+
   it('ignores lines that are not stats', () => {
     expect(parseStatsLine('>f+++++++++ a.txt', emptyStats())).toBe(false)
   })

packages/rsync-core/src/parse.ts

@@ -100,13 +100,29 @@ export function classifyItemize(itemize: string): ChangeAction {
   return 'update'
 }
 
-const STATS_PATTERNS: Array<[keyof RsyncStats, RegExp]> = [
+/**
+ * Counts. rsync prints these with `comma_num`, so they are exact integers
+ * whatever `--human-readable` is doing.
+ */
+const COUNT_PATTERNS: Array<[keyof RsyncStats, RegExp]> = [
   ['filesTotal', /^Number of files:\s+([\d,]+)/],
   ['filesTransferred', /^Number of (?:regular files transferred|files transferred):\s+([\d,]+)/],
-  ['literalBytes', /^Literal data:\s+([\d,]+)/],
-  ['matchedBytes', /^Matched data:\s+([\d,]+)/],
-  ['totalBytesSent', /^Total bytes sent:\s+([\d,]+)/],
-  ['totalBytesReceived', /^Total bytes received:\s+([\d,]+)/],
+  // Protocol 33 (rsync 3.5.0+) prints this between "Literal data" and
+  // "Matched data". Older rsync omits the line entirely.
+  ['logicalBlocksTouched', /^Number of 4 KiB logical blocks touched:\s+([\d,]+)/],
+]
+
+/**
+ * Byte totals. rsync prints these with `human_num`, so under
+ * `--human-readable` -- which DiskPush passes by default -- they arrive as
+ * "3.50M" rather than "3,500,000". The unit suffix has to be read back or the
+ * value truncates to its leading digit.
+ */
+const BYTE_PATTERNS: Array<[keyof RsyncStats, RegExp]> = [
+  ['literalBytes', /^Literal data:\s+([\d,.]+)([KMGTP])?/],
+  ['matchedBytes', /^Matched data:\s+([\d,.]+)([KMGTP])?/],
+  ['totalBytesSent', /^Total bytes sent:\s+([\d,.]+)([KMGTP])?/],
+  ['totalBytesReceived', /^Total bytes received:\s+([\d,.]+)([KMGTP])?/],
 ]
 
 export function emptyStats(): RsyncStats {
@@ -116,21 +132,29 @@ export function emptyStats(): RsyncStats {
     totalBytesSent: null,
     totalBytesReceived: null,
     literalBytes: null,
+    logicalBlocksTouched: null,
     matchedBytes: null,
     speedup: null,
   }
 }
 
 /** Folds one `--stats` line into an accumulating stats object. */
 export function parseStatsLine(line: string, into: RsyncStats): boolean {
-  for (const [key, pattern] of STATS_PATTERNS) {
+  for (const [key, pattern] of COUNT_PATTERNS) {
     const match = pattern.exec(line)
     if (match) {
       into[key] = Number((match[1] ?? '').replaceAll(',', ''))
       return true
     }
   }
-  const speedup = /^(?:total size is [\d,]+\s+)?speedup is ([\d.]+)/i.exec(line)
+  for (const [key, pattern] of BYTE_PATTERNS) {
+    const match = pattern.exec(line)
+    if (match) {
+      into[key] = toNumber(match[1] ?? '0', match[2])
+      return true
+    }
+  }
+  const speedup = /^(?:total size is [\d,.]+[KMGTP]?\s+)?speedup is ([\d.]+)/i.exec(line)
   if (speedup) {
     into.speedup = Number(speedup[1])
     return true

packages/rsync-core/src/version.test.ts

@@ -14,6 +14,20 @@ Checksum list:
 Compress list:
     zstd lz4 zlibx zlib none`
 
+const RSYNC_351 = `rsync  version 3.5.1  protocol version 33
+Copyright (C) 1996-2026 by Andrew Tridgell, Wayne Davison, and others.
+Capabilities:
+    64-bit files, 64-bit inums, 64-bit timestamps, 64-bit long ints,
+    socketpairs, symlinks, symtimes, hardlinks, hardlink-specials,
+    hardlink-symlinks, IPv6, atimes, batchfiles, inplace, append, ACLs,
+    xattrs, optional secluded-args, iconv, prealloc, stop-at, no crtimes
+Optimizations:
+    SIMD-roll, no asm-roll, openssl-crypto, no asm-MD5
+Checksum list:
+    xxh128 xxh3 xxh64 md5 md4 sha1 none
+Compress list:
+    zstd lz4 zlibx zlib none`
+
 const RSYNC_316 = `rsync  version 3.1.6  protocol version 31
 Capabilities:
     64-bit files, socketpairs, hardlinks, symlinks, ACLs, xattrs, protect-args
@@ -29,6 +43,10 @@ describe('parseRsyncVersion', () => {
     expect(parseRsyncVersion(RSYNC_341)).toMatchObject({ major: 3, minor: 4, patch: 1 })
   })
 
+  it('reads 3.5.1, the current release', () => {
+    expect(parseRsyncVersion(RSYNC_351)).toMatchObject({ major: 3, minor: 5, patch: 1, raw: '3.5.1' })
+  })
+
   it('defaults a missing patch to zero', () => {
     expect(parseRsyncVersion('rsync  version 3.2  protocol version 31')).toMatchObject({ major: 3, minor: 2, patch: 0 })
   })
@@ -67,6 +85,27 @@ describe('parseRsyncCapabilities', () => {
     expect(parseRsyncCapabilities(RSYNC_341).mkpath).toBe(true)
     expect(parseRsyncCapabilities(RSYNC_316).mkpath).toBe(false)
   })
+
+  it('reads protocol 33 from 3.5.1 and keeps every older capability', () => {
+    const caps = parseRsyncCapabilities(RSYNC_351)
+    expect(caps.protocol).toBe(33)
+    expect(caps.secludedArgsByDefault).toBe(true)
+    expect(caps.mkpath).toBe(true)
+    expect(caps.zstd).toBe(true)
+    expect(caps.acls).toBe(true)
+    expect(caps.xattrs).toBe(true)
+  })
+
+  it('gates the 4 KiB block stat on protocol 33', () => {
+    expect(parseRsyncCapabilities(RSYNC_351).blockStats).toBe(true)
+    expect(parseRsyncCapabilities(RSYNC_341).blockStats).toBe(false)
+    expect(parseRsyncCapabilities(RSYNC_316).blockStats).toBe(false)
+  })
+
+  it('falls back to the version when the banner prints no protocol line', () => {
+    expect(parseRsyncCapabilities('rsync  version 3.5.1').blockStats).toBe(true)
+    expect(parseRsyncCapabilities('rsync  version 3.4.1').blockStats).toBe(false)
+  })
 })
 
 describe('atLeast', () => {
@@ -82,6 +121,13 @@ describe('atLeast', () => {
 })
 
 describe('intersectCapabilities', () => {
+  it('drops the block stat unless both ends speak protocol 33', () => {
+    const modern = parseRsyncCapabilities(RSYNC_351)
+    expect(intersectCapabilities(modern, parseRsyncCapabilities(RSYNC_341)).blockStats).toBe(false)
+    expect(intersectCapabilities(modern, modern).blockStats).toBe(true)
+    expect(intersectCapabilities(modern, parseRsyncCapabilities(RSYNC_341)).protocol).toBe(32)
+  })
+
   it('takes the weaker of the two ends', () => {
     const merged = intersectCapabilities(parseRsyncCapabilities(RSYNC_341), parseRsyncCapabilities(RSYNC_316))
     expect(merged.zstd).toBe(false)

packages/rsync-core/src/version.ts

@@ -18,6 +18,11 @@ export type RsyncCapabilities = {
   secludedArgsAvailable: boolean
   /** `--mkpath` (3.2.3+). */
   mkpath: boolean
+  /**
+   * `--stats` reports the number of 4 KiB logical blocks touched. This rides on
+   * protocol 33, which rsync 3.5.0 introduced, and both ends must negotiate it.
+   */
+  blockStats: boolean
   acls: boolean
   xattrs: boolean
   hardLinks: boolean
@@ -30,6 +35,7 @@ const UNKNOWN: RsyncCapabilities = {
   secludedArgsByDefault: false,
   secludedArgsAvailable: false,
   mkpath: false,
+  blockStats: false,
   acls: false,
   xattrs: false,
   hardLinks: true,
@@ -68,18 +74,22 @@ export function parseRsyncCapabilities(banner: string): RsyncCapabilities {
 
   const lower = banner.toLowerCase()
   const protocolMatch = /protocol\s+version\s+(\d+)/i.exec(banner)
+  const protocol = protocolMatch ? Number(protocolMatch[1]) : null
 
   // The compress list is authoritative for zstd; the version only makes it possible.
   const compressList = /compress list:\s*([^\n]*)/i.exec(lower)
   const zstd = compressList ? /\bzstd\b/.test(compressList[1] ?? '') : false
 
   return {
     version,
-    protocol: protocolMatch ? Number(protocolMatch[1]) : null,
+    protocol,
     zstd,
     secludedArgsByDefault: atLeast(version, 3, 2, 4),
     secludedArgsAvailable: atLeast(version, 3, 0, 0),
     mkpath: atLeast(version, 3, 2, 3),
+    // The negotiated protocol is what actually decides this; the version is
+    // only a fallback for a banner that does not print a protocol line.
+    blockStats: protocol === null ? atLeast(version, 3, 5, 0) : protocol >= 33,
     acls: /\bACLs\b/i.test(banner) && !/no ACLs/i.test(banner),
     xattrs: /\bxattrs\b/i.test(banner) && !/no xattrs/i.test(banner),
     hardLinks: !/no hardlinks/i.test(lower),
@@ -105,6 +115,7 @@ export function intersectCapabilities(a: RsyncCapabilities, b: RsyncCapabilities
     secludedArgsByDefault: a.secludedArgsByDefault && b.secludedArgsByDefault,
     secludedArgsAvailable: a.secludedArgsAvailable && b.secludedArgsAvailable,
     mkpath: a.mkpath && b.mkpath,
+    blockStats: a.blockStats && b.blockStats,
     acls: a.acls && b.acls,
     xattrs: a.xattrs && b.xattrs,
     hardLinks: a.hardLinks && b.hardLinks,

packages/schemas/src/events.ts

@@ -18,6 +18,12 @@ export type RsyncStats = {
   totalBytesSent: number | null
   totalBytesReceived: number | null
   literalBytes: number | null
+  /**
+   * Distinct 4 KiB logical file blocks the receiver touched. rsync prints this
+   * only when both ends negotiate protocol 33 (rsync 3.5.0+), so it stays null
+   * against anything older rather than being reported as zero.
+   */
+  logicalBlocksTouched: number | null
   matchedBytes: number | null
   speedup: number | null
 }

tests/live/rsync-behaviour.test.ts

@@ -224,3 +224,34 @@ describe.skipIf(!hasRsync)('live rsync: resume', () => {
     expect(readFileSync(join(dst, 'big.bin')).equals(big)).toBe(true)
   }, 60_000)
 })
+
+describe.skipIf(!hasRsync)('live rsync: --stats against the installed binary', () => {
+  it('reads byte totals back at full size rather than truncating them', async () => {
+    const { src, dst } = tree('stats-bytes')
+    // Large enough that rsync renders the totals as "3.50M" under
+    // --human-readable, which DiskPush passes by default.
+    writeFileSync(join(src, 'blob.bin'), Buffer.alloc(3_500_000, 7))
+
+    const result = await sync(src, dst)
+    expect(result.ok).toBe(true)
+    expect(result.stats).not.toBeNull()
+    expect(result.stats?.totalBytesSent).toBeGreaterThan(3_000_000)
+    expect(result.stats?.literalBytes).toBeGreaterThan(3_000_000)
+    expect(result.stats?.speedup).toBeGreaterThan(0)
+  }, 60_000)
+
+  it('reports 4 KiB blocks touched exactly when this rsync speaks protocol 33', async () => {
+    const { src, dst } = tree('stats-blocks')
+    writeFileSync(join(src, 'blob.bin'), Buffer.alloc(200_000, 3))
+
+    const result = await sync(src, dst)
+    expect(result.ok).toBe(true)
+    // rsync 3.5.0+ prints the stat; anything older omits the line, and the
+    // field stays null rather than being reported as zero.
+    if (capabilities.blockStats) {
+      expect(result.stats?.logicalBlocksTouched).toBeGreaterThan(0)
+    } else {
+      expect(result.stats?.logicalBlocksTouched).toBeNull()
+    }
+  }, 60_000)
+})