feat: read result pages from object storage links

src/services/RemoteWorkerSimulationService.ts

@@ -54,6 +54,7 @@ import { taskTotals } from '../util/taskTotals';
 import { ValidateShape } from '../util/Types';
 import { SimulationSourceType } from '../WrapperApp/components/Simulation/RunSimulationForm';
 import CacheMap from './CacheMap';
+import { resolveStoredPages } from './storedPages';
 
 const CACHEABLE_STATUSES = new Set([
 	StatusState.FAILED,
@@ -383,6 +384,11 @@ export default class RemoteWorkerSimulationService implements SimulationService
 					})
 					.json<ResponseGetJobResults>();
 
+				await Promise.all(
+					response.estimators.map(async estimator => {
+						estimator.pages = await resolveStoredPages(estimator.pages, signal);
+					})
+				);
 				updateEstimators(response.estimators);
 
 				const jobInputs = await this.getJobInputs(info, signal, cache);
@@ -475,7 +481,7 @@ export default class RemoteWorkerSimulationService implements SimulationService
 				const estimator: Estimator[] = [
 					{
 						name: estimatorName,
-						pages: response.pages
+						pages: await resolveStoredPages(response.pages, signal)
 					}
 				];
 

src/services/storedPages.test.ts

@@ -0,0 +1,30 @@
+import ky from 'ky';
+
+import { Page } from '../JsRoot/GraphData';
+import { resolveStoredPages } from './storedPages';
+
+jest.mock('ky', () => ({ get: jest.fn() }));
+
+describe('resolveStoredPages', () => {
+	it('fetches pages kept in storage and converts their keys, keeping pages sent inline', async () => {
+		(ky.get as jest.Mock).mockReturnValue({
+			json: () =>
+				Promise.resolve({ metadata: { page_number: 1 }, axis_dim1: { values: [1] } })
+		});
+		const inline = { dimensions: 0, metadata: { pageNumber: 0 } } as unknown as Page;
+		const stored = {
+			pageNumber: 1,
+			url: 'https://storage/p1',
+			format: 'page-json-gzip-v1'
+		} as unknown as Page;
+
+		const pages = await resolveStoredPages([inline, stored]);
+
+		expect(pages[0]).toBe(inline);
+		expect(pages[1]).toEqual({ metadata: { pageNumber: 1 }, axisDim1: { values: [1] } });
+		expect(ky.get).toHaveBeenCalledWith(
+			'https://storage/p1',
+			expect.objectContaining({ retry: 2 })
+		);
+	});
+});

src/services/storedPages.ts

@@ -0,0 +1,31 @@
+import ky from 'ky';
+
+import { Page } from '../JsRoot/GraphData';
+import { snakeToCamelCase } from '../util/Notation/Notation';
+
+/**
+ * A result page kept in object storage: the backend sends a signed link instead of the page.
+ */
+type StoredPage = { pageNumber: number; url: string; format: string };
+
+const isStoredPage = (page: unknown): page is StoredPage =>
+	typeof (page as StoredPage | undefined)?.url === 'string';
+
+/**
+ * Replaces pages kept in object storage with the pages themselves, fetched in parallel from their signed links.
+ * The page JSON goes through the same snake_case to camelCase conversion as the backend's API responses.
+ */
+export function resolveStoredPages(pages: Page[], signal?: AbortSignal): Promise<Page[]> {
+	return Promise.all(
+		pages.map(page =>
+			isStoredPage(page)
+				? ky
+						// no timeout: requests beyond the browser's ~6 connections wait in its queue, and a large page
+						// on a slow link takes long; the abort signal still stops them
+						.get(page.url, { signal, retry: 2, timeout: false })
+						.json<object>()
+						.then(json => snakeToCamelCase(json, true) as unknown as Page)
+				: page
+		)
+	);
+}