diff --git a/src/libs/converter b/src/libs/converter index 10223d4b9..47791d417 160000 --- a/src/libs/converter +++ b/src/libs/converter @@ -1 +1 @@ -Subproject commit 10223d4b9916650b8c6b9b6a5f9d7d9ecefe6106 +Subproject commit 47791d41729f01e8241b9f2a28e7c98b99dcaca8 diff --git a/src/services/RemoteWorkerSimulationService.ts b/src/services/RemoteWorkerSimulationService.ts index b9ea16836..aae84f68d 100644 --- a/src/services/RemoteWorkerSimulationService.ts +++ b/src/services/RemoteWorkerSimulationService.ts @@ -54,6 +54,7 @@ import { taskTotals } from '../util/taskTotals'; import { ValidateShape } from '../util/Types'; import { SimulationSourceType } from '../WrapperApp/components/Simulation/RunSimulationForm'; import CacheMap from './CacheMap'; +import { resolveStoredPages } from './storedPages'; const CACHEABLE_STATUSES = new Set([ StatusState.FAILED, @@ -383,6 +384,11 @@ export default class RemoteWorkerSimulationService implements SimulationService }) .json(); + await Promise.all( + response.estimators.map(async estimator => { + estimator.pages = await resolveStoredPages(estimator.pages, signal); + }) + ); updateEstimators(response.estimators); const jobInputs = await this.getJobInputs(info, signal, cache); @@ -475,7 +481,7 @@ export default class RemoteWorkerSimulationService implements SimulationService const estimator: Estimator[] = [ { name: estimatorName, - pages: response.pages + pages: await resolveStoredPages(response.pages, signal) } ]; diff --git a/src/services/storedPages.test.ts b/src/services/storedPages.test.ts new file mode 100644 index 000000000..d13014935 --- /dev/null +++ b/src/services/storedPages.test.ts @@ -0,0 +1,30 @@ +import ky from 'ky'; + +import { Page } from '../JsRoot/GraphData'; +import { resolveStoredPages } from './storedPages'; + +jest.mock('ky', () => ({ get: jest.fn() })); + +describe('resolveStoredPages', () => { + it('fetches pages kept in storage and converts their keys, keeping pages sent inline', async () => { + (ky.get as jest.Mock).mockReturnValue({ + json: () => + Promise.resolve({ metadata: { page_number: 1 }, axis_dim1: { values: [1] } }) + }); + const inline = { dimensions: 0, metadata: { pageNumber: 0 } } as unknown as Page; + const stored = { + pageNumber: 1, + url: 'https://storage/p1', + format: 'page-json-gzip-v1' + } as unknown as Page; + + const pages = await resolveStoredPages([inline, stored]); + + expect(pages[0]).toBe(inline); + expect(pages[1]).toEqual({ metadata: { pageNumber: 1 }, axisDim1: { values: [1] } }); + expect(ky.get).toHaveBeenCalledWith( + 'https://storage/p1', + expect.objectContaining({ retry: 2 }) + ); + }); +}); diff --git a/src/services/storedPages.ts b/src/services/storedPages.ts new file mode 100644 index 000000000..113329ab6 --- /dev/null +++ b/src/services/storedPages.ts @@ -0,0 +1,31 @@ +import ky from 'ky'; + +import { Page } from '../JsRoot/GraphData'; +import { snakeToCamelCase } from '../util/Notation/Notation'; + +/** + * A result page kept in object storage: the backend sends a signed link instead of the page. + */ +type StoredPage = { pageNumber: number; url: string; format: string }; + +const isStoredPage = (page: unknown): page is StoredPage => + typeof (page as StoredPage | undefined)?.url === 'string'; + +/** + * Replaces pages kept in object storage with the pages themselves, fetched in parallel from their signed links. + * The page JSON goes through the same snake_case to camelCase conversion as the backend's API responses. + */ +export function resolveStoredPages(pages: Page[], signal?: AbortSignal): Promise { + return Promise.all( + pages.map(page => + isStoredPage(page) + ? ky + // no timeout: requests beyond the browser's ~6 connections wait in its queue, and a large page + // on a slow link takes long; the abort signal still stops them + .get(page.url, { signal, retry: 2, timeout: false }) + .json() + .then(json => snakeToCamelCase(json, true) as unknown as Page) + : page + ) + ); +}