Skip to content

add progress bar and status for preparing datasets #2435

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
cnuart wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

add progress bar and status for preparing datasets #2435

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 63 additions & 26 deletions src/WrapperApp/components/Simulation/SimulationCard/SimulationCardContent.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { useEffect, useState } from 'react';
import {
currentJobStatusData,
currentTaskStatusData,
DatasetPreparationInfo,
JobUnknownStatus,
SimulationInfo,
StatusState,
Expand Down Expand Up @@ -102,6 +103,41 @@ export function SimulationProgress(props: {
);
}

export function SimulationPreparation(props: { preparation: DatasetPreparationInfo }) {
const { stage, fromCache, completedCount, totalCount, activeDataset, overallProgress } =
props.preparation;

const showCounter = totalCount > 0 && completedCount > 0;
const counter = showCounter ? ` (${completedCount}/${totalCount})` : '';
const suffix = activeDataset ? ` — ${activeDataset}` : '';

const label =
stage === 'initializing'
? 'Initializing simulation engine'
: stage === 'preparing'
? 'Preparing simulation'
: fromCache
? `Loading datasets from cache${counter}${suffix}`
: `Preparing datasets${counter}${suffix}`;

const determinate = stage === 'loading' && overallProgress !== undefined;

return (
<>
<Typography
color='textDisabled'
variant='caption'>
{label}
</Typography>
<LinearProgress
variant={determinate ? 'determinate' : 'indeterminate'}
value={determinate ? overallProgress! * 100 : undefined}
sx={{ height: 4, mb: 1, mt: 0.5 }}
/>
</>
);
}

export const SimulationCardContent = ({
duration,
endTime,
Expand Down Expand Up @@ -166,32 +202,33 @@ export const SimulationCardContent = ({
))}
</Box>
<Box sx={{ minHeight: '40px' }}>
{needsProgressBar(simulationStatus.jobState)
? currentJobStatusData['hasSpecificProperty'](
simulationStatus,
'jobTasksStatus'
) && (
<SimulationProgress
formatedStartDate={formatedStartDate}
duration={duration}
simulationStatus={simulationStatus}
/>
)
: simulationStatus.jobState === StatusState.COMPLETED && (
<>
<Typography color='textDisabled'>
Start: {formatedStartDate}
</Typography>
{endTime && (
<Typography color='textDisabled'>
End: {formatedEndDate}
</Typography>
)}
<Typography color='textDisabled'>
Duration: {millisecondsToTimeString(duration)}
</Typography>
</>
)}
{simulationStatus.jobState === StatusState.PENDING &&
simulationStatus.datasetPreparation ? (
<SimulationPreparation preparation={simulationStatus.datasetPreparation} />
) : needsProgressBar(simulationStatus.jobState) ? (
currentJobStatusData['hasSpecificProperty'](
simulationStatus,
'jobTasksStatus'
) && (
<SimulationProgress
formatedStartDate={formatedStartDate}
duration={duration}
simulationStatus={simulationStatus}
/>
)
) : (
simulationStatus.jobState === StatusState.COMPLETED && (
<>
<Typography color='textDisabled'>Start: {formatedStartDate}</Typography>
{endTime && (
<Typography color='textDisabled'>End: {formatedEndDate}</Typography>
)}
<Typography color='textDisabled'>
Duration: {millisecondsToTimeString(duration)}
</Typography>
</>
)
)}
</Box>
</CardContent>
);
Expand Down
202 changes: 156 additions & 46 deletions src/services/Geant4LocalWorkerSimulationService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { GEANT4_DATASETS, TOTAL_DATASET_SIZE_MB } from '../Geant4Worker/Geant4DatasetCacheService';
import Geant4Worker from '../Geant4Worker/Geant4Worker';
import { Estimator, Page1D, Page2D } from '../JsRoot/GraphData';
import { PythonConverterContext } from '../PythonConverter/PythonConverterService';
Expand All @@ -19,6 +20,7 @@ import {
SimulatorType
} from '../types/RequestTypes';
import {
DatasetPreparationInfo,
EstimatorItem,
Geant4InputFilesNames,
InputFilesRecord,
Expand All @@ -38,6 +40,8 @@ type JobId = string;
interface JobMetadata {
title: string;
inputType: SimulationSourceType;
phase: 'initializing' | 'loading' | 'preparing';
datasetType?: Geant4DatasetsType;
}

export default class Geant4LocalWorkerSimulationService implements SimulationService {
Expand Down Expand Up @@ -105,7 +109,12 @@ export default class Geant4LocalWorkerSimulationService implements SimulationSer

const worker = new Geant4Worker();
this.workers[jobId] = worker;
this.jobsMetadata[jobId] = { title: title ?? '', inputType };
this.jobsMetadata[jobId] = {
title: title ?? '',
inputType,
phase: 'initializing',
datasetType: geant4DatasetType
};

this.inputFiles[jobId] = {
// @ts-ignore
Expand All @@ -124,12 +133,16 @@ export default class Geant4LocalWorkerSimulationService implements SimulationSer
this.numPrimaries = rawNumPrimaries ? parseInt(rawNumPrimaries) : 0;

worker.init().then(async () => {
if (geant4DatasetType && geant4DatasetType === Geant4DatasetsType.FULL) {
this.jobsMetadata[jobId].phase = 'loading';

if (geant4DatasetType === Geant4DatasetsType.FULL) {
await worker.loadDeps();
} else {
await worker.loadDepsLazy();
}

this.jobsMetadata[jobId].phase = 'preparing';

// @ts-ignore
await worker.includeFile('geometry.gdml', simData['geometry.gdml']);
// @ts-ignore
Expand Down Expand Up @@ -179,12 +192,97 @@ export default class Geant4LocalWorkerSimulationService implements SimulationSer
];
}

private async getDatasetPreparation(
jobId: string
): Promise<DatasetPreparationInfo | undefined> {
const worker = this.workers[jobId];
const metadata = this.jobsMetadata[jobId];

if (!worker || !metadata || worker.getState() !== StatusState.PENDING) {
return undefined;
}

const fromCache = metadata.datasetType === Geant4DatasetsType.FULL;
const isFullDataset = metadata.datasetType === Geant4DatasetsType.FULL;
const totalCount = isFullDataset ? GEANT4_DATASETS.length : 0;

if (metadata.phase === 'initializing') {
return {
stage: 'initializing',
fromCache,
completedCount: 0,
totalCount: 0
};
}

if (metadata.phase === 'preparing') {
return {
stage: 'preparing',
fromCache,
completedCount: 0,
totalCount
};
}

const base: DatasetPreparationInfo = {
stage: 'loading',
fromCache,
completedCount: 0,
totalCount
};

if (!isFullDataset || !worker.getIsInitialized()) {
return base;
}

try {
const progress = await worker.pollDatasetProgress();

if (!progress) {
return base;
}

let completedCount = 0;
let activeDataset: string | undefined;
let weightedProgress = 0;

for (const dataset of GEANT4_DATASETS) {
const entry = progress[dataset.name];

if (!entry) {
continue;
}

if (entry.stage === 'done') {
completedCount += 1;
} else if (!activeDataset) {
activeDataset = dataset.name;
}

weightedProgress += entry.progress * dataset.approximateSizeMB;
}

return {
...base,
completedCount,
activeDataset,
overallProgress:
TOTAL_DATASET_SIZE_MB > 0 ? weightedProgress / TOTAL_DATASET_SIZE_MB : undefined
};
} catch {
return base;
}
}

async getJobStatus(...args: RequestGetJobStatus): Promise<JobStatusData | undefined> {
const [info, cache = true, beforeCacheWrite, signal] = args;
const { jobId } = info;

if (this.responseCache.jobStatusData.hasOwnProperty(info.jobId)) {
return this.responseCache.jobStatusData[info.jobId];
const cached = this.responseCache.jobStatusData[info.jobId];
cached.datasetPreparation = await this.getDatasetPreparation(jobId);

return cached;
}

if (!this.workers.hasOwnProperty(info.jobId)) {
Expand All @@ -203,7 +301,8 @@ export default class Geant4LocalWorkerSimulationService implements SimulationSer
platform: 'DIRECT'
},
jobState: this.workers[jobId].getState(),
jobTasksStatus: this.getJobTasksStatus(jobId)
jobTasksStatus: this.getJobTasksStatus(jobId),
datasetPreparation: await this.getDatasetPreparation(jobId)
} as JobStatusData;

if (cache) beforeCacheWrite?.(jobId, this.responseCache.jobStatusData[info.jobId]);
Expand Down Expand Up @@ -417,6 +516,7 @@ export default class Geant4LocalWorkerSimulationService implements SimulationSer
platform: 'DIRECT'
},
jobTasksStatus: this.getJobTasksStatus(jobId),
datasetPreparation: await this.getDatasetPreparation(jobId),
input: {
inputType: this.jobsMetadata[jobId].inputType,
inputFiles: this.inputFiles[jobId] as InputFilesRecord<Geant4InputFilesNames, ''>,
Expand Down Expand Up @@ -472,23 +572,30 @@ export default class Geant4LocalWorkerSimulationService implements SimulationSer
pageSize * pageIdx
);

const simulations = await Promise.all(
paginatedWorkersEntries.map(
async ([jobId, worker]): Promise<JobStatusData> => ({
jobId,
title: this.jobsMetadata[jobId]?.title,
startTime: worker.getStartTime().toString(),
endTime: worker.getEndTime()?.toString(),
metadata: {
inputType: this.jobsMetadata[jobId].inputType,
simType: 'Geant4',
server: '',
platform: 'DIRECT'
},
jobState: worker.getState(),
jobTasksStatus: this.getJobTasksStatus(jobId),
datasetPreparation: await this.getDatasetPreparation(jobId)
})
)
);

return {
pageCount: Math.ceil(filteredWorkersEntries.length / pageSize),
simulationsCount: filteredWorkersEntries.length,
simulations: paginatedWorkersEntries.map(([jobId, worker]) => ({
jobId,
title: this.jobsMetadata[jobId]?.title,
startTime: worker.getStartTime().toString(),
endTime: worker.getEndTime()?.toString(),
metadata: {
inputType: this.jobsMetadata[jobId].inputType,
simType: 'Geant4',
server: '',
platform: 'DIRECT'
},
jobState: worker.getState(),
jobTasksStatus: this.getJobTasksStatus(jobId)
})),
simulations,
message: ''
};
}
Expand All @@ -505,34 +612,37 @@ export default class Geant4LocalWorkerSimulationService implements SimulationSer
)
.filter(Boolean);

return workersEntries.map(([jobId, worker]) => {
if (this.responseCache.pageStatus.hasOwnProperty(jobId)) {
return this.responseCache.pageStatus[jobId];
}

const response = {
jobId,
title: this.jobsMetadata[jobId]?.title,
startTime: worker.getStartTime().toString(),
endTime: worker.getEndTime()?.toString(),
metadata: {
inputType: this.jobsMetadata[jobId].inputType,
simType: 'Geant4',
server: '',
platform: 'DIRECT'
},
jobState: worker.getState(),
jobTasksStatus: this.getJobTasksStatus(jobId)
} as JobStatusData;

if (worker.getState() === StatusState.COMPLETED) {
this.responseCache.pageStatus[jobId] = response;

if (cache) beforeCacheWrite?.(jobId, response);
}

return response;
});
return Promise.all(
workersEntries.map(async ([jobId, worker]) => {
if (this.responseCache.pageStatus.hasOwnProperty(jobId)) {
return this.responseCache.pageStatus[jobId];
}

const response = {
jobId,
title: this.jobsMetadata[jobId]?.title,
startTime: worker.getStartTime().toString(),
endTime: worker.getEndTime()?.toString(),
metadata: {
inputType: this.jobsMetadata[jobId].inputType,
simType: 'Geant4',
server: '',
platform: 'DIRECT'
},
jobState: worker.getState(),
jobTasksStatus: this.getJobTasksStatus(jobId),
datasetPreparation: await this.getDatasetPreparation(jobId)
} as JobStatusData;

if (worker.getState() === StatusState.COMPLETED) {
this.responseCache.pageStatus[jobId] = response;

if (cache) beforeCacheWrite?.(jobId, response);
}

return response;
})
);
}

cancelJob(info: SimulationInfo, signal?: AbortSignal | undefined): Promise<void> {
Expand Down
Loading
Loading