diff --git a/README.md b/README.md index c3f03ba..5b5d08b 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ required for the default simulations. - [Architecture](docs/reference/sfv_reference_architecture_summary.md): implementation boundaries and shared execution rules. - [Technical plan](docs/reference/sfv_technical_implementation_plan.md): implemented functionality and current work items. - [Validation](docs/VALIDATION.md): repeatable scenarios, distinct replay and independent-prediction claims, and cache tests. +- [Co-simulation](docs/CO_SIMULATION.md): run SFV sessions against an external network simulator over JSON lines. The included media fixtures and dash.js-derived patch need a separate provenance/licensing review before redistribution. Raw browser recordings, diff --git a/docs/CO_SIMULATION.md b/docs/CO_SIMULATION.md new file mode 100644 index 0000000..724d13e --- /dev/null +++ b/docs/CO_SIMULATION.md @@ -0,0 +1,82 @@ +# Co-simulation with an external network + +An external harness can run SFV sessions against its own network simulator, for example the [VQEG CAP–CSP collaboration pilot](https://github.com/VQEG/cap-csp-collaboration-test-pilot) with the FikoRE 5G emulator. The harness owns simulation time and byte delivery. SFV owns playback, ABR decisions, swipes and session records. + +The harness starts `scripts/cosim-network-step-worker.mjs` as a child process and exchanges one JSON object per line on stdin and stdout. Every request gets exactly one reply. A failure is answered with `{"type": "error", "message": "..."}` and leaves the process running. + +## Components + +The worker wraps two classes in `src/cosim/`, both exported from `src/index.js`: + +- `SFVNetworkStepHarness` runs one `SFVSimulationController` per UE on a shared clock, applies each network step to all UEs, collects their network actions, and drains CAP reports. +- `SFVExternalTransport` is the transport of each controller. It turns requests and aborts into `submit_request` and `cancel_request` actions and keeps the media metadata of each request on the player side. The external network only sees an opaque request ID and a byte count. + +## Messages + +`initialize` creates the harness and is answered with `{"type": "initialized"}`: + +```json +{ + "type": "initialize", + "config": { + "dataset": { "videos": [] }, + "sessions": [ + { + "ue_id": 0, + "player_behavior": "simple", + "config": { "sessionStartTimeS": 0, "sessionEndTimeS": 30 }, + "rule_config": {} + } + ], + "timingDefaults": {}, + "duration_s": 30, + "network_backend": "transport_fikore", + "run_id": "example", + "signaling_level": "L0", + "condition": "C1", + "swipe_profile": "scripted" + } +} +``` + +`player_behavior` selects `SFVCurrentVideoRule` (`simple`, B1) or `SFVMultiVideoPrefetchRule` (`preload`, B2). `duration_s` and `network_backend` are required. + +`network_step` advances all sessions to `step.time_s` and applies the step's download events. The first step must be at `t = 0`, times must increase, and the step with `is_final: true` must be at `duration_s`: + +```json +{ + "type": "network_step", + "step": { + "time_s": 0.5, + "is_final": false, + "events": [ + { + "event_type": "DownloadProgress", + "ue_id": 0, + "request_id": "ue0-r1", + "time_s": 0.5, + "bytes_delivered": 12000 + } + ], + "collaboration": { "csp_by_ue": { "0": { "plan_downlink_mbps": 10 } } } + } +} +``` + +Event types are `DownloadProgress` and `DownloadCancelled` (with cumulative `bytes_delivered`) and `DownloadCompleted`. The optional `collaboration.csp_by_ue` object carries the CSP fields each UE may see at the current signaling level. The harness passes them to `applyCspUpdate` only when a UE's fields change. + +A non-final step is answered with the network actions to execute before the next step: + +```json +{ + "type": "player_actions", + "actions": [ + { "op": "submit_request", "ue_id": 0, "request_id": "ue0-r2", "bytes_total": 52129 }, + { "op": "cancel_request", "ue_id": 0, "request_id": "ue0-r1" } + ] +} +``` + +The final step is answered with `{"type": "player_step_result", "actions": [], "result": {...}}`, where `result.sessions` holds the session record of each UE. Requests still open at that point are recorded as cancelled. + +`cap_reports` is answered with `{"type": "cap_reports", "reports": {"0": {...}}}`. A UE appears only when `drainCapReports` has a report due: on a stall edge, a video switch or a buffer change of at least 1.0 s, at most once per second of session time. After the final step the rate limit does not apply, so the terminal state is always reported. diff --git a/scripts/cosim-network-step-worker.mjs b/scripts/cosim-network-step-worker.mjs new file mode 100755 index 0000000..35f20d9 --- /dev/null +++ b/scripts/cosim-network-step-worker.mjs @@ -0,0 +1,28 @@ +#!/usr/bin/env node +// Co-simulation player process: one JSON message per line on stdin, one reply per line on stdout. +// The protocol is described in docs/CO_SIMULATION.md. +import readline from 'node:readline'; +import { SFVNetworkStepHarness } from '../src/cosim/SFVNetworkStepHarness.js'; + +const input = readline.createInterface({ input: process.stdin, crlfDelay: Infinity }); +let harness; +for await (const line of input) { + try { + const message = JSON.parse(line); + let reply; + if (message.type === 'initialize') { + if (harness) throw new Error('Player already initialized'); + harness = new SFVNetworkStepHarness(message.config); + reply = { type: 'initialized' }; + } else if (!harness) throw new Error('Player not initialized'); + else if (message.type === 'network_step') reply = harness.onStep(message.step); + else if (message.type === 'cap_reports') + reply = { type: 'cap_reports', reports: harness.getCapReports() }; + else throw new Error(`Unknown IPC message ${message.type}`); + process.stdout.write(JSON.stringify(reply) + '\n'); + } catch (error) { + process.stdout.write( + JSON.stringify({ type: 'error', message: String(error.message ?? error) }) + '\n' + ); + } +} diff --git a/src/cosim/SFVExternalTransport.js b/src/cosim/SFVExternalTransport.js new file mode 100644 index 0000000..4f4cf66 --- /dev/null +++ b/src/cosim/SFVExternalTransport.js @@ -0,0 +1,152 @@ +import { REQUEST_STATE } from '../core/constants.js'; + +const ID = /^[A-Za-z0-9_.-]{1,64}$/; + +/** The player-facing transport knows only the generic NetworkBackend contract. + * Media metadata stays in this registry; only opaque request identifiers and byte + * counts are returned to the external harness. It never advances global time. + */ +export class SFVExternalTransport { + constructor({ ue_id, queueAction, now }) { + if ( + !Number.isSafeInteger(ue_id) || + ue_id < 0 || + typeof queueAction !== 'function' || + typeof now !== 'function' + ) { + throw new TypeError('Invalid external transport configuration'); + } + this.ue_id = ue_id; + this.queueAction = queueAction; + this.now = now; + this.requests = new Map(); + this.used = new Set(); + } + + start(request, now = this.now()) { + const { requestId, bytesTotal } = request; + if ( + !ID.test(requestId) || + this.used.has(requestId) || + !Number.isSafeInteger(bytesTotal) || + bytesTotal <= 0 + ) { + throw new TypeError('Invalid or reused player request ID'); + } + const entry = { + ...structuredClone(request), + state: REQUEST_STATE.ACTIVE, + startTime: now, + firstByteTime: null, + endTime: null, + bytesDelivered: 0, + finalBytesDelivered: null, + abortRequestedAt: null, + bytesDeliveredAtAbort: null + }; + this.used.add(requestId); + this.requests.set(requestId, entry); + this.queueAction({ + op: 'submit_request', + ue_id: this.ue_id, + request_id: requestId, + bytes_total: bytesTotal + }); + return structuredClone(entry); + } + + abort(requestId, now = this.now()) { + const request = this.requests.get(requestId); + if (!request || ![REQUEST_STATE.ACTIVE, REQUEST_STATE.CANCELLING].includes(request.state)) + return false; + if (request.state === REQUEST_STATE.CANCELLING) return true; + request.state = REQUEST_STATE.CANCELLING; + request.abortRequestedAt = now; + request.bytesDeliveredAtAbort = request.bytesDelivered; + this.queueAction({ op: 'cancel_request', ue_id: this.ue_id, request_id: requestId }); + return true; + } + + get(requestId) { + const entry = this.requests.get(requestId); + return entry ? structuredClone(entry) : null; + } + + list() { + return [...this.requests.values()].map(entry => structuredClone(entry)); + } + + /** Apply an event from NetworkStep; completion and cancellation are terminal. */ + receiveEvent(event, dispatch) { + const { event_type: kind, request_id: requestId, time_s: time } = event; + const entry = this.requests.get(requestId); + if (!entry || ![REQUEST_STATE.ACTIVE, REQUEST_STATE.CANCELLING].includes(entry.state)) { + throw new Error(`Unknown or terminal network request ${requestId}`); + } + if (kind === 'DownloadProgress') { + this.#progress(entry, event.bytes_delivered, time, dispatch); + } else if (kind === 'DownloadCompleted') { + this.#progress(entry, entry.bytesTotal, time, dispatch); + entry.state = REQUEST_STATE.COMPLETED; + entry.endTime = time; + entry.finalBytesDelivered = entry.bytesTotal; + dispatch({ eventType: 'REQUEST_COMPLETED', requestId, time, totalBytes: entry.bytesTotal }); + } else if (kind === 'DownloadCancelled') { + if (entry.state !== REQUEST_STATE.CANCELLING) + throw new Error(`Unexpected cancellation of ${requestId}`); + this.#progress(entry, event.bytes_delivered, time, dispatch); + entry.state = REQUEST_STATE.CANCELLED; + entry.endTime = time; + entry.finalBytesDelivered = event.bytes_delivered; + dispatch({ + eventType: 'REQUEST_CANCELLED', + requestId, + time, + totalBytes: event.bytes_delivered + }); + } else { + throw new Error(`Unsupported request event ${kind}`); + } + } + + #progress(entry, amount, time, dispatch) { + if ( + !Number.isSafeInteger(amount) || + amount < entry.bytesDelivered || + amount > entry.bytesTotal + ) { + throw new Error(`Invalid cumulative bytes for ${entry.requestId}`); + } + const delta = amount - entry.bytesDelivered; + if (delta && entry.firstByteTime === null) { + entry.firstByteTime = time; + dispatch({ eventType: 'REQUEST_FIRST_BYTE', requestId: entry.requestId, time }); + } + entry.bytesDelivered = amount; + if (delta) + dispatch({ + eventType: 'REQUEST_PROGRESS', + requestId: entry.requestId, + time, + deltaBytes: delta, + totalBytes: amount + }); + } + + finish(time) { + const events = []; + for (const entry of this.requests.values()) { + if (![REQUEST_STATE.ACTIVE, REQUEST_STATE.CANCELLING].includes(entry.state)) continue; + entry.state = REQUEST_STATE.CANCELLED; + entry.endTime = time; + entry.finalBytesDelivered = entry.bytesDelivered; + events.push({ + eventType: 'REQUEST_CANCELLED', + requestId: entry.requestId, + time, + totalBytes: entry.bytesDelivered + }); + } + return events; + } +} diff --git a/src/cosim/SFVNetworkStepHarness.js b/src/cosim/SFVNetworkStepHarness.js new file mode 100644 index 0000000..1cf06b7 --- /dev/null +++ b/src/cosim/SFVNetworkStepHarness.js @@ -0,0 +1,223 @@ +import { SFVSimulationController } from '../simulation/SFVSimulationController.js'; +import { SFVCurrentVideoRule } from '../rules/SFVCurrentVideoRule.js'; +import { SFVMultiVideoPrefetchRule } from '../rules/SFVMultiVideoPrefetchRule.js'; +import { SFVExternalTransport } from './SFVExternalTransport.js'; + +const EPS = 1e-8; +const EVENT_TYPES = new Set(['DownloadProgress', 'DownloadCompleted', 'DownloadCancelled']); + +/** Shared-time SFV sessions behind the NetworkBackend API, without FikoRE wire semantics. + * The external harness supplies a whole NetworkStep and receives opaque network actions only after + * all UEs have processed the same step. No policy or simulated network is duplicated. + */ +export class SFVNetworkStepHarness { + constructor({ + dataset, + sessions, + timingDefaults = {}, + run_id = 'sfv-network-run', + signaling_level = 'L0', + condition = 'unspecified', + swipe_profile = 'scripted', + duration_s = 1, + network_backend + } = {}) { + if ( + !dataset?.videos?.length || + !Array.isArray(sessions) || + !sessions.length || + !Number.isFinite(duration_s) || + duration_s <= 0 + ) + throw new TypeError('Missing dataset, sessions or duration'); + if (typeof network_backend !== 'string' || !network_backend) + throw new TypeError('Missing network_backend name'); + this.time = 0; + this.sequence = -1; + this.ended = false; + this.actions = []; + this.states = new Map(); + this.run_id = run_id; + this.signaling_level = signaling_level; + this.duration_s = duration_s; + this.network_backend = network_backend; + for (const [index, session] of sessions.entries()) { + const ue_id = session.ue_id ?? session.config?.ueId ?? index; + if (!Number.isSafeInteger(ue_id) || ue_id < 0 || this.states.has(ue_id)) + throw new TypeError('Duplicate or invalid UE'); + const behavior = + session.player_behavior ?? (session.abr?.name === 'prefetch' ? 'preload' : 'simple'); + if (!['simple', 'preload'].includes(behavior)) + throw new TypeError('Unsupported player behavior'); + const Rule = behavior === 'simple' ? SFVCurrentVideoRule : SFVMultiVideoPrefetchRule; + const transport = new SFVExternalTransport({ + ue_id, + now: () => this.time, + queueAction: action => { + if (!this.ended) this.actions.push(action); + } + }); + const config = { + sessionId: session.config?.sessionId ?? `${run_id}-ue${ue_id}`, + ueId: ue_id, + ...timingDefaults, + ...session.config, + maxTimeS: duration_s, + runId: run_id, + condition, + signalingLevel: signaling_level, + swipeProfile: swipe_profile, + playerBehavior: behavior, + transport: { + capacityMbps: null, + latencyMs: null, + cancelTailBytes: null, + cancelTailMs: null, + network_backend + }, + networkBackend: network_backend, + rule: structuredClone( + session.rule_config ?? session.abr?.config ?? session.config?.rule ?? {} + ) + }; + const controller = new SFVSimulationController({ + dataset, + rule: new Rule(), + config, + transport, + requestIdPrefix: `ue${ue_id}-r` + }); + this.states.set(ue_id, { + controller, + transport, + started: false, + done: false, + result: null, + lastCspKey: undefined + }); + } + } + + onStep(step) { + if (this.ended) throw new Error('Run already finished'); + if ( + !step || + !Number.isFinite(step.time_s) || + step.time_s < this.time - EPS || + step.time_s > this.duration_s + EPS || + !Array.isArray(step.events) || + typeof step.is_final !== 'boolean' + ) + throw new TypeError('Invalid NetworkStep'); + if (this.sequence === -1 && Math.abs(step.time_s) > EPS) + throw new Error('Initial NetworkStep must be t=0'); + if (this.sequence >= 0 && step.time_s <= this.time + EPS) + throw new Error('NetworkStep time must progress'); + if (step.is_final && Math.abs(step.time_s - this.duration_s) > EPS) { + throw new Error('Final step does not match experiment duration'); + } + this.sequence++; + this.time = step.time_s; + this.#applyCollaborationSnapshot(step); + // Advance every UE to the same backend timestamp before applying any network events. + for (const state of this.states.values()) { + if (state.started && !state.done) state.controller.advancePlaybackTo(this.time); + } + for (const event of step.events) { + if ( + !event || + !EVENT_TYPES.has(event.event_type) || + !Number.isSafeInteger(event.ue_id) || + !this.states.has(event.ue_id) || + !Number.isFinite(event.time_s) || + event.time_s > this.time + EPS || + event.time_s < 0 + ) { + throw new TypeError('Invalid NetworkStep event'); + } + const state = this.states.get(event.ue_id); + if (!state.started) throw new Error('Network event for unstarted session'); + state.transport.receiveEvent(event, e => state.controller.applyNetworkEvent(e)); + } + for (const state of this.states.values()) { + const controller = state.controller; + if ( + !state.started && + !state.done && + controller.sessionStartTimeS <= this.time + EPS && + controller.sessionStartTimeS < controller.sessionEndTimeS - EPS + ) { + controller.startAt(this.time); + state.started = true; + } + if (state.started && !state.done) { + while (controller.nextSwipeTime() <= this.time + EPS) { + if (!controller.applySwipeAt(controller.nextSwipeTime())) + throw new Error('Swipe schedule stalled'); + } + if (controller.sessionEndTimeS <= this.time + EPS || step.is_final) { + controller.stopQueuedRequests(this.time, true); + state.done = true; + } + } + } + if (!step.is_final) { + for (const state of this.states.values()) { + if (!state.started || state.done) continue; + for (let attempt = 0; attempt < 20; attempt++) { + state.controller.decideAtEpoch(); + if (!state.controller.hasPendingDecision()) break; + if (attempt === 19) throw new Error('Decision epoch guard exceeded'); + } + } + } + const actions = this.actions.splice(0); + if (step.is_final) { + // No new network operations may be submitted after the last NetworkStep. + for (const state of this.states.values()) { + if (state.started) { + for (const event of state.transport.finish(this.time)) + state.controller.applyNetworkEvent(event); + state.result = state.controller.finishAt(this.time, true); + } else state.result = state.controller.finishAt(this.time, false); + } + this.ended = true; + return { + type: 'player_step_result', + actions: [], + result: { + run_id: this.run_id, + network_backend: this.network_backend, + sim_time_s: this.time, + sessions: [...this.states].map(([ue_id, state]) => ({ ue_id, result: state.result })) + } + }; + } + return { type: 'player_actions', actions }; + } + + #applyCollaborationSnapshot(step) { + const byUe = step.collaboration?.csp_by_ue; + if (byUe == null || typeof byUe !== 'object') return; + for (const [ue_id, state] of this.states) { + if (state.done) continue; + const fields = byUe[String(ue_id)] ?? null; + const key = JSON.stringify(fields); + if (key === state.lastCspKey) continue; + state.lastCspKey = key; + state.controller.applyCspUpdate(fields, this.time); + } + } + + /** Pending CAP reports per UE (event-driven, ≤1/s); ``{}`` when nothing changed. + * After the final step the drain bypasses the rate limit so terminal state is reported. */ + getCapReports() { + const reports = {}; + for (const [ue_id, state] of this.states) { + if (!state.started) continue; + const drained = state.controller.drainCapReports(this.time, { final: this.ended }); + if (drained.length) reports[String(ue_id)] = drained.at(-1); + } + return reports; + } +} diff --git a/src/index.js b/src/index.js index adf7d0d..0acf1f8 100644 --- a/src/index.js +++ b/src/index.js @@ -29,6 +29,8 @@ export { resolveUserInteractions } from './simulation/generators/SFVGenerators.js'; export { SFVSingleUEQueuedTransport } from './simulation/SFVSingleUEQueuedTransport.js'; +export { SFVExternalTransport } from './cosim/SFVExternalTransport.js'; +export { SFVNetworkStepHarness } from './cosim/SFVNetworkStepHarness.js'; export { SFV_EXPERIMENT_SCHEMA_VERSION, diff --git a/test/cosim-network-step-harness.test.js b/test/cosim-network-step-harness.test.js new file mode 100644 index 0000000..04aafc0 --- /dev/null +++ b/test/cosim-network-step-harness.test.js @@ -0,0 +1,153 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import readline from 'node:readline'; +import { SFVNetworkStepHarness } from '../src/index.js'; + +const dataset = JSON.parse( + fs.readFileSync(new URL('../fixtures/content/content.json', import.meta.url)) +); + +function createHarness(overrides = {}) { + return new SFVNetworkStepHarness({ + dataset, + sessions: [ + { + ue_id: 0, + player_behavior: 'simple', + config: { sessionStartTimeS: 0, sessionEndTimeS: 4, swipeTimesS: [] } + } + ], + duration_s: 4, + network_backend: 'test', + ...overrides + }); +} + +function step(time_s, events = [], extra = {}) { + return { time_s, events, is_final: false, ...extra }; +} + +function completions(actions, time_s) { + return actions + .filter(action => action.op === 'submit_request') + .map(action => ({ + event_type: 'DownloadCompleted', + ue_id: action.ue_id, + request_id: action.request_id, + time_s + })); +} + +test('harness submits requests, consumes completions and returns session results', () => { + const harness = createHarness(); + let reply = harness.onStep(step(0)); + assert.equal(reply.type, 'player_actions'); + assert.ok(reply.actions.some(action => action.op === 'submit_request')); + for (let t = 0.5; t < 4; t += 0.5) { + reply = harness.onStep(step(t, completions(reply.actions, t))); + assert.equal(reply.type, 'player_actions'); + } + const final = harness.onStep({ time_s: 4, events: [], is_final: true }); + assert.equal(final.type, 'player_step_result'); + assert.deepEqual(final.actions, []); + assert.equal(final.result.network_backend, 'test'); + assert.deepEqual( + final.result.sessions.map(session => session.ue_id), + [0] + ); + assert.ok(final.result.sessions[0].result); + assert.throws(() => harness.onStep(step(4)), /already finished/); +}); + +test('harness rejects invalid step sequences', () => { + assert.throws(() => createHarness().onStep(step(0.5)), /must be t=0/); + const harness = createHarness(); + harness.onStep(step(0)); + assert.throws(() => harness.onStep(step(0)), /must progress/); + assert.throws( + () => harness.onStep({ time_s: 1, events: [], is_final: true }), + /does not match experiment duration/ + ); + assert.throws( + () => harness.onStep(step(1, [{ event_type: 'DownloadCompleted', ue_id: 7, time_s: 1 }])), + /Invalid NetworkStep event/ + ); +}); + +test('harness applies a CSP snapshot only when it changes', () => { + const harness = createHarness(); + const { controller } = harness.states.get(0); + const updates = []; + const original = controller.applyCspUpdate.bind(controller); + controller.applyCspUpdate = (fields, timeS) => { + updates.push({ fields, timeS }); + return original(fields, timeS); + }; + const fields = { plan_downlink_mbps: 10 }; + harness.onStep(step(0, [], { collaboration: { csp_by_ue: { 0: fields } } })); + harness.onStep(step(0.5, [], { collaboration: { csp_by_ue: { 0: fields } } })); + harness.onStep(step(1, [], { collaboration: { csp_by_ue: { 0: { plan_downlink_mbps: 5 } } } })); + harness.onStep(step(1.5)); + assert.deepEqual( + updates.map(update => [update.timeS, update.fields.plan_downlink_mbps]), + [ + [0, 10], + [1, 5] + ] + ); +}); + +test('harness drains CAP reports per started UE', () => { + const harness = createHarness({ + sessions: [ + { + ue_id: 0, + player_behavior: 'simple', + config: { sessionStartTimeS: 0, sessionEndTimeS: 4, swipeTimesS: [] } + }, + { + ue_id: 1, + player_behavior: 'simple', + config: { sessionStartTimeS: 2, sessionEndTimeS: 4, swipeTimesS: [] } + } + ] + }); + harness.onStep(step(0)); + const reports = harness.getCapReports(); + assert.deepEqual(Object.keys(reports), ['0']); + assert.equal(reports['0'].current_video_id, 'v1'); + assert.deepEqual(harness.getCapReports(), {}); +}); + +test('worker answers line-delimited JSON messages', async () => { + const worker = spawn( + process.execPath, + [new URL('../scripts/cosim-network-step-worker.mjs', import.meta.url).pathname], + { stdio: ['pipe', 'pipe', 'inherit'] } + ); + const lines = readline.createInterface({ input: worker.stdout })[Symbol.asyncIterator](); + async function exchange(message) { + worker.stdin.write(JSON.stringify(message) + '\n'); + return JSON.parse((await lines.next()).value); + } + try { + assert.deepEqual(await exchange({ type: 'network_step', step: step(0) }), { + type: 'error', + message: 'Player not initialized' + }); + const config = { + dataset, + sessions: [{ ue_id: 0, config: { sessionStartTimeS: 0, sessionEndTimeS: 1 } }], + duration_s: 1, + network_backend: 'test' + }; + assert.deepEqual(await exchange({ type: 'initialize', config }), { type: 'initialized' }); + assert.equal((await exchange({ type: 'network_step', step: step(0) })).type, 'player_actions'); + assert.equal((await exchange({ type: 'cap_reports' })).type, 'cap_reports'); + assert.match((await exchange({ type: 'unknown' })).message, /Unknown IPC message/); + } finally { + worker.stdin.end(); + } +});