From fcf90b4122d4411f3cb33979ba166dedd9c65cc5 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Mon, 17 Aug 2026 09:50:28 +0000 Subject: [PATCH 1/2] feat(example): adds additional visr recon workflow example --- src/copier_template/pyproject.toml.jinja | 2 +- .../templates/visr_recon.txt.jinja | 278 +++++++++++++ .../create_notebook_in_image.py.jinja | 2 +- .../visr-recon.ipynb.jinja | 367 +++++++++++++++++ .../templates/visr_recon.txt | 276 +++++++++++++ .../visr_notebook_example/visr-recon.ipynb | 372 ++++++++++++++++++ 6 files changed, 1295 insertions(+), 2 deletions(-) create mode 100644 src/copier_template/src/{{ project_name }}/templates/visr_recon.txt.jinja create mode 100644 src/copier_template/src/{{ project_name }}/workflow_definitions/visr_notebook_example/visr-recon.ipynb.jinja create mode 100644 src/python_interface_to_workflows/templates/visr_recon.txt create mode 100644 src/python_interface_to_workflows/workflow_definitions/visr_notebook_example/visr-recon.ipynb diff --git a/src/copier_template/pyproject.toml.jinja b/src/copier_template/pyproject.toml.jinja index 6266061..50e2194 100644 --- a/src/copier_template/pyproject.toml.jinja +++ b/src/copier_template/pyproject.toml.jinja @@ -160,7 +160,7 @@ description = "Run tests with coverage" commands = [ [ "pytest", - "--cov=python_interface_to_workflows", + "--cov={% endraw %}{{repo_name}}{% raw %}", "--cov-report", "term", "--cov-report", diff --git a/src/copier_template/src/{{ project_name }}/templates/visr_recon.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/visr_recon.txt.jinja new file mode 100644 index 0000000..55c77e6 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/templates/visr_recon.txt.jinja @@ -0,0 +1,278 @@ +{% raw %} +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: visr-recon-with-{% endraw %}{{repo_name}}{% raw %} + annotations: + workflows.argoproj.io/description: |- + ViSR recon + example.yaml + workflows.argoproj.io/title: ViSR recon + workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %} + labels: + workflows.diamond.ac.uk/science-group-imaging: 'true' +spec: + entrypoint: workflowentry + templates: + - name: workflowentry + dag: + tasks: + - name: tomo-recon + template: tomo-recon + arguments: + parameters: + - name: config + value: |- + [ + { + "method": "standard_tomo", + "module_path": "httomo.data.hdf.loaders", + "parameters": { + "data_path": "/entry1/tomo_entry/data/data", + "image_key_path": "/entry1/tomo_entry/instrument/detector/image_key", + "rotation_angles": { + "data_path": "/entry1/tomo_entry/data/rotation_angle" + }, + "preview": { + "detector_y": { + "start": 100, + "stop": 102 + } + } + } + }, + { + "method": "remove_outlier", + "module_path": "tomopy.misc.corr", + "parameters": { + "dif": 0.1, + "size": 3, + "axis": "auto" + } + }, + { + "method": "dark_flat_field_correction", + "module_path": "httomolibgpu.prep.normalize", + "parameters": { + "flats_multiplier": 1, + "darks_multiplier": 1 + } + }, + { + "method": "find_center_vo", + "module_path": "httomolibgpu.recon.rotation", + "parameters": { + "ind": null, + "smin": -50, + "smax": 50, + "srad": 6, + "step": 0.25, + "ratio": 0.5, + "drop": 20 + }, + "id": "centering", + "side_outputs": { + "cor": "centre_of_rotation" + } + }, + { + "method": "FBP3d_tomobar", + "module_path": "httomolibgpu.recon.algorithm", + "parameters": { + "center": "${{centering.side_outputs.centre_of_rotation}}", + "filter_freq_cutoff": 0.6, + "recon_size": null, + "recon_mask_radius": null + }, + "save_result": true + }, + { + "method": "calculate_stats", + "module_path": "httomo.methods", + "parameters": {}, + "id": "statistics", + "side_outputs": { + "glob_stats": "glob_stats" + } + }, + { + "method": "rescale_to_int", + "module_path": "httomolib.misc.rescale", + "parameters": { + "perc_range_min": 0, + "perc_range_max": 100, + "bits": 8, + "glob_stats": "${{statistics.side_outputs.glob_stats}}" + }, + "save_result": true + } + ] + - name: input + value: '/dls/i12/data/2025/.......' + - name: output + value: '/dls/i12/data/2025/.......' + - name: recon_outdir_name + value: sweep-run + - name: nprocs + value: '1' + - name: memory + value: 1Gi + - name: convert-recon-data-format + depends: tomo-recon + template: convert-recon-data-format + arguments: + parameters: + - name: recon_dir_path + value: '{{tasks.tomo-recon.outputs.parameters.out-path}}' + - name: tomo-recon + podSpecPatch: '{"containers": [{"name": "main", "resources": {"limits": {"cpu": + "{{inputs.parameters.nprocs}}", "memory": "{{inputs.parameters.memory}}", "nvidia.com/gpu": + "{{inputs.parameters.nprocs}}"}, "requests": {"cpu": "{{inputs.parameters.nprocs}}", + "memory": "{{inputs.parameters.memory}}", "nvidia.com/gpu": "{{inputs.parameters.nprocs}}"}}}]}' + tolerations: + - effect: NoSchedule + key: nvidia.com/gpu + operator: Exists + - effect: NoSchedule + key: nodetype + operator: Equal + value: gpu + - effect: NoSchedule + key: nodegroup + operator: Equal + value: workflows + inputs: + parameters: + - name: config + - name: input + - name: output + - name: recon_outdir_name + - name: nprocs + - name: memory + outputs: + parameters: + - name: out-path + valueFrom: + path: /tmp/parameters.json + script: + image: ghcr.io/diamondlightsource/httomo:latest + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: config = json.loads(r'''{{inputs.parameters.config}}''') + except: config = r'''{{inputs.parameters.config}}''' + try: input = json.loads(r'''{{inputs.parameters.input}}''') + except: input = r'''{{inputs.parameters.input}}''' + try: memory = json.loads(r'''{{inputs.parameters.memory}}''') + except: memory = r'''{{inputs.parameters.memory}}''' + try: nprocs = json.loads(r'''{{inputs.parameters.nprocs}}''') + except: nprocs = r'''{{inputs.parameters.nprocs}}''' + try: output = json.loads(r'''{{inputs.parameters.output}}''') + except: output = r'''{{inputs.parameters.output}}''' + try: recon_outdir_name = json.loads(r'''{{inputs.parameters.recon_outdir_name}}''') + except: recon_outdir_name = r'''{{inputs.parameters.recon_outdir_name}}''' + + import json + import subprocess + loaded_config = json.dumps(config) + subprocess.check_call(['/opt/conda/bin/mpirun', '-n', str(nprocs), '/opt/conda/bin/python', '-m', 'httomo', 'run', '--pipeline-format', 'json', '--output-folder-name', recon_outdir_name, input, loaded_config, output]) + with open('/tmp/parameters.json', 'w') as f: + json.dump(f'{output}/{recon_outdir_name}', f) + command: + - /opt/conda/bin/python + env: + - name: CUPY_CACHE_DIR + value: /tmp/.cupy/kernel_cache + - name: MKL_NUM_THREADS + value: '1' + - name: NUMEXPR_NUM_THREADS + value: '1' + - name: OMP_NUM_THREADS + value: '1' + volumeMounts: + - name: session + mountPath: '{{workflow.parameters.visitdir}}' + - name: tmpdir + mountPath: /tmp + - name: convert-recon-data-format + inputs: + parameters: + - name: recon_dir_path + - name: tmpdir_path + default: /tmp + - name: raw_recon_filename + default: recon.raw + - name: metadata_filename + default: metadata.json + outputs: + artifacts: + - name: recon + path: '{{inputs.parameters.tmpdir_path}}/{{inputs.parameters.raw_recon_filename}}' + archive: + none: {} + - name: metadata + path: '{{inputs.parameters.tmpdir_path}}/{{inputs.parameters.metadata_filename}}' + archive: + none: {} + script: + image: ghcr.io/diamondlightsource/httomo:latest + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: metadata_filename = json.loads(r'''{{inputs.parameters.metadata_filename}}''') + except: metadata_filename = r'''{{inputs.parameters.metadata_filename}}''' + try: raw_recon_filename = json.loads(r'''{{inputs.parameters.raw_recon_filename}}''') + except: raw_recon_filename = r'''{{inputs.parameters.raw_recon_filename}}''' + try: recon_dir_path = json.loads(r'''{{inputs.parameters.recon_dir_path}}''') + except: recon_dir_path = r'''{{inputs.parameters.recon_dir_path}}''' + try: tmpdir_path = json.loads(r'''{{inputs.parameters.tmpdir_path}}''') + except: tmpdir_path = r'''{{inputs.parameters.tmpdir_path}}''' + + import json + from pathlib import Path + import h5py + RAW_RECON_PATH = f'{tmpdir_path}/{raw_recon_filename}' + HDF5_RECON_DIR = Path(recon_dir_path) + HDF5_RECON_FILENAME_PATTERN = '*-httomolib-rescale_to_int.h5' + hdf5_recon_data_path = list(HDF5_RECON_DIR.glob(HDF5_RECON_FILENAME_PATTERN))[0] + with h5py.File(hdf5_recon_data_path, 'r') as f: + data = f['/data'][:] + data.tofile(RAW_RECON_PATH) + METADATA_PATH = f'{tmpdir_path}/{metadata_filename}' + order = 'C' if data.flags.c_contiguous else 'F' + metadata = {'shape': list(data.shape), 'dtype': str(data.dtype), 'order': order} + with open(METADATA_PATH, 'w') as f: + f.write(json.dumps(metadata, indent=2)) + command: + - /opt/conda/bin/python + volumeMounts: + - name: session + mountPath: '{{workflow.parameters.visitdir}}' + - name: tmpdir + mountPath: /tmp + volumeClaimTemplates: + - metadata: + name: tmpdir + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + volumes: + - name: session + hostPath: + path: '{{workflow.parameters.visitdir}}' + type: Directory + arguments: + parameters: + - name: visitdir + valueFrom: + configMapKeyRef: + name: sessionspaces + key: data_directory +{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja index d305429..d209786 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja @@ -18,7 +18,7 @@ from hera.workflows.archive import NoneArchiveStrategy # and was created with the included dockerfile. global_config.set_class_defaults( # pyright: ignore Script, - image="ghcr.io/diamondlightsource/python-interface-to-workflows-mounted-image:latest", + image="ghcr.io/diamondlightsource/{% endraw %}{{repo_name}}{% raw %}-mounted-image:latest", ) diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/visr_notebook_example/visr-recon.ipynb.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/visr_notebook_example/visr-recon.ipynb.jinja new file mode 100644 index 0000000..c7ac4e2 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/visr_notebook_example/visr-recon.ipynb.jinja @@ -0,0 +1,367 @@ +{% raw %} +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "a4baf3e4", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from hera.shared import global_config\n", + "from hera.workflows import (\n", + " DAG,\n", + " Artifact,\n", + " Parameter,\n", + " Script,\n", + " Volume,\n", + " Workflow,\n", + " Env,\n", + " script, # pyright: ignore[reportUnknownVariableType]\n", + ")\n", + "from hera.workflows import models as m\n", + "from hera.workflows.archive import NoneArchiveStrategy\n", + "\n", + "# Sets the default image, unless specified otherwise, to this.\n", + "global_config.set_class_defaults( # pyright: ignore\n", + " Script, image=\"ghcr.io/diamondlightsource/httomo:latest\"\n", + ")" + "# Do note! This workflow will not run, as the DLS paths have been replaced with\n" + "# placeholders, so as not to fill up required directories.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "98316b4f", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "# The script decorator allows hera to convert python code into yaml\n", + "@script(\n", + "# Assure that we only run on a pod with certain parameters\n", + " pod_spec_patch=json.dumps({\"containers\":\n", + " [{\"name\":\"main\",\n", + " \"resources\":\n", + " {\"limits\":{\"cpu\":\"{{inputs.parameters.nprocs}}\",\n", + " \"memory\":\"{{inputs.parameters.memory}}\",\n", + " \"nvidia.com/gpu\":\"{{inputs.parameters.nprocs}}\"},\n", + " \"requests\":{\"cpu\":\"{{inputs.parameters.nprocs}}\",\n", + " \"memory\":\"{{inputs.parameters.memory}}\",\n", + " \"nvidia.com/gpu\":\"{{inputs.parameters.nprocs}}\"}}}]}),\n", + "\n", + "# Assure that we only run on a pod with certain parameters\n", + " tolerations=[\n", + " m.Toleration(key=\"nvidia.com/gpu\",operator=\"Exists\",effect=\"NoSchedule\"),\n", + " m.Toleration(key=\"nodetype\",operator=\"Equal\",value=\"gpu\",effect=\"NoSchedule\"),\n", + " m.Toleration(key=\"nodegroup\",operator=\"Equal\",value=\"workflows\",effect=\"NoSchedule\")],\n", + "# Inform the pod which installation of python we intend to run on\n", + " command=[\"/opt/conda/bin/python\"],\n", + "# mount both volumes - one being a parameters in the diamond file system, the other\n", + "# being a mounted file path\n", + " volume_mounts=[\n", + " m.VolumeMount(name=\"session\", mount_path=\"{{workflow.parameters.visitdir}}\"),\n", + " m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\"),\n", + " ],\n", + "# inform the next step that the file we produce here will be used as a parameter for the\n", + "# next script\n", + " outputs=[\n", + " Parameter(\n", + " name=\"out-path\",\n", + " value_from=m.ValueFrom(\n", + " path=\"/tmp/parameters.json\"\n", + " )\n", + " )\n", + " ],\n", + "# establish some environmental variables shared between steps.\n", + " env=[\n", + " Env(name=\"CUPY_CACHE_DIR\", value=\"/tmp/.cupy/kernel_cache\"),\n", + " Env(name=\"MKL_NUM_THREADS\", value=\"1\"),\n", + " Env(name=\"NUMEXPR_NUM_THREADS\", value=\"1\"),\n", + " Env(name=\"OMP_NUM_THREADS\", value=\"1\"),\n", + " ]\n", + ")\n", + "def tomo_recon(\n", + " config: str, input: str, output: str, recon_outdir_name: str, nprocs: int, memory: str\n", + "):\n", + " import json\n", + " import subprocess\n", + "# load the config string as a json - otherwise, in the yaml conversion, this is loaded\n", + "# as a string instead, which prevents mpirun from functioning.\n", + " loaded_config = json.dumps(config)\n", + "\n", + " subprocess.check_call([\n", + " \"/opt/conda/bin/mpirun\",\n", + " \"-n\",\n", + " str(nprocs),\n", + " \"/opt/conda/bin/python\",\n", + " \"-m\",\n", + " \"httomo\",\n", + " \"run\",\n", + " \"--pipeline-format\",\n", + " \"json\",\n", + " \"--output-folder-name\",\n", + " recon_outdir_name,\n", + " input,\n", + " loaded_config,\n", + " output\n", + " ])\n", + "\n", + " with open(\"/tmp/parameters.json\", \"w\") as f:\n", + " json.dump(f\"{output}/{recon_outdir_name}\", f)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1535543f", + "metadata": {}, + "outputs": [], + "source": [ + "@script(\n", + " command=[\"/opt/conda/bin/python\"],\n", + " volume_mounts=[\n", + " m.VolumeMount(name=\"session\", mount_path=\"{{workflow.parameters.visitdir}}\"),\n", + " m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\"),\n", + " ],\n", + " #our outputs this time are artifacts, that we do not want compressed.\n", + " outputs=[\n", + " Artifact(\n", + " name=\"recon\",\n", + " path=\"{{inputs.parameters.tmpdir_path}}/{{inputs.parameters.raw_recon_filename}}\",\n", + " archive=NoneArchiveStrategy(),\n", + " ),\n", + " Artifact(\n", + " name=\"metadata\",\n", + " path=\"{{inputs.parameters.tmpdir_path}}/{{inputs.parameters.metadata_filename}}\",\n", + " archive=NoneArchiveStrategy(),\n", + " ),\n", + " ]\n", + ")\n", + "def convert_recon_data_format(recon_dir_path: str,\n", + " tmpdir_path: str = \"/tmp\",\n", + " raw_recon_filename: str = \"recon.raw\",\n", + " metadata_filename: str = \"metadata.json\"):\n", + " import json\n", + " from pathlib import Path\n", + "\n", + " import h5py\n", + "\n", + "\n", + "\n", + " RAW_RECON_PATH = f\"{tmpdir_path}/{raw_recon_filename}\"\n", + " HDF5_RECON_DIR = Path(recon_dir_path)\n", + " HDF5_RECON_FILENAME_PATTERN = \"*-httomolib-rescale_to_int.h5\"\n", + " hdf5_recon_data_path = list(HDF5_RECON_DIR.glob(HDF5_RECON_FILENAME_PATTERN))[0]\n", + "\n", + " with h5py.File(hdf5_recon_data_path, \"r\") as f:\n", + " data = f[\"/data\"][:]\n", + " data.tofile(RAW_RECON_PATH)\n", + "\n", + " METADATA_PATH = f\"{tmpdir_path}/{metadata_filename}\"\n", + "\n", + " order = \"C\" if data.flags.c_contiguous else \"F\"\n", + " metadata = {\"shape\": list(data.shape), \"dtype\": str(data.dtype), \"order\": order}\n", + " with open(METADATA_PATH, \"w\") as f:\n", + " f.write(json.dumps(metadata, indent=2))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7fdf9da0", + "metadata": {}, + "outputs": [], + "source": [ + "from hera.workflows.volume import HostPathVolume\n", + "\n", + "with Workflow(\n", + " # name of workflow - will append a short identifier automatically.\n", + " name=\"visr-recon-with-{% endraw %}{{repo_name}}{% raw %}\",\n", + " # the following is the same as for writing any yaml workflow, but as variables.\n", + " # All of these are required aside from \"workflows.argoproj.io/description\".\n", + " entrypoint=\"workflowentry\",\n", + " api_version=\"argoproj.io/v1alpha1\",\n", + " kind=\"WorkflowTemplate\",\n", + " labels={\"workflows.diamond.ac.uk/science-group-imaging\": \"true\"},\n", + " annotations={\n", + " \"workflows.argoproj.io/title\": \"ViSR recon\",\n", + " \"workflows.argoproj.io/description\": \"\"\"ViSR recon\n", + "example.yaml\"\"\",\n", + " \"workflows.diamond.ac.uk/repository\": \"https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}\",\n", + " },\n", + " # mounts the diamond file system\n", + " volumes=[\n", + " Volume(name=\"tmpdir\", mount_path=\"/tmp/\", size=\"1Gi\"),\n", + " HostPathVolume(name=\"session\",\n", + " path=\"{{workflow.parameters.visitdir}}\",\n", + " type=\"Directory\"),\n", + " ],\n", + " arguments=m.Arguments(\n", + " parameters=[\n", + " m.Parameter(\n", + " name=\"visitdir\",\n", + " value_from=m.ValueFrom(\n", + " config_map_key_ref=m.ConfigMapKeySelector(\n", + " name=\"sessionspaces\",\n", + " key=\"data_directory\"\n", + " )\n", + " )\n", + " )\n", + " ]\n", + " )\n", + ") as w:\n", + " with DAG(name=\"workflowentry\"):\n", + " # Alternatively, we could have this in /mounted_files/ and run\n", + " # a simple step where we copy over this json, before running\n", + " # the step in the httomo image\n", + " config = \"\"\"[\n", + " {\n", + " \"method\": \"standard_tomo\",\n", + " \"module_path\": \"httomo.data.hdf.loaders\",\n", + " \"parameters\": {\n", + " \"data_path\": \"/entry1/tomo_entry/data/data\",\n", + " \"image_key_path\": \"/entry1/tomo_entry/instrument/detector/image_key\",\n", + " \"rotation_angles\": {\n", + " \"data_path\": \"/entry1/tomo_entry/data/rotation_angle\"\n", + " },\n", + " \"preview\": {\n", + " \"detector_y\": {\n", + " \"start\": 100,\n", + " \"stop\": 102\n", + " }\n", + " }\n", + " }\n", + " },\n", + " {\n", + " \"method\": \"remove_outlier\",\n", + " \"module_path\": \"tomopy.misc.corr\",\n", + " \"parameters\": {\n", + " \"dif\": 0.1,\n", + " \"size\": 3,\n", + " \"axis\": \"auto\"\n", + " }\n", + " },\n", + " {\n", + " \"method\": \"dark_flat_field_correction\",\n", + " \"module_path\": \"httomolibgpu.prep.normalize\",\n", + " \"parameters\": {\n", + " \"flats_multiplier\": 1,\n", + " \"darks_multiplier\": 1\n", + " }\n", + " },\n", + " {\n", + " \"method\": \"find_center_vo\",\n", + " \"module_path\": \"httomolibgpu.recon.rotation\",\n", + " \"parameters\": {\n", + " \"ind\": null,\n", + " \"smin\": -50,\n", + " \"smax\": 50,\n", + " \"srad\": 6,\n", + " \"step\": 0.25,\n", + " \"ratio\": 0.5,\n", + " \"drop\": 20\n", + " },\n", + " \"id\": \"centering\",\n", + " \"side_outputs\": {\n", + " \"cor\": \"centre_of_rotation\"\n", + " }\n", + " },\n", + " {\n", + " \"method\": \"FBP3d_tomobar\",\n", + " \"module_path\": \"httomolibgpu.recon.algorithm\",\n", + " \"parameters\": {\n", + " \"center\": \"${{centering.side_outputs.centre_of_rotation}}\",\n", + " \"filter_freq_cutoff\": 0.6,\n", + " \"recon_size\": null,\n", + " \"recon_mask_radius\": null\n", + " },\n", + " \"save_result\": true\n", + " },\n", + " {\n", + " \"method\": \"calculate_stats\",\n", + " \"module_path\": \"httomo.methods\",\n", + " \"parameters\": {},\n", + " \"id\": \"statistics\",\n", + " \"side_outputs\": {\n", + " \"glob_stats\": \"glob_stats\"\n", + " }\n", + " },\n", + " {\n", + " \"method\": \"rescale_to_int\",\n", + " \"module_path\": \"httomolib.misc.rescale\",\n", + " \"parameters\": {\n", + " \"perc_range_min\": 0,\n", + " \"perc_range_max\": 100,\n", + " \"bits\": 8,\n", + " \"glob_stats\": \"${{statistics.side_outputs.glob_stats}}\"\n", + " },\n", + " \"save_result\": true\n", + " }\n", + "]\"\"\"\n", + " recon = tomo_recon(\n", + " arguments={\n", + " \"config\": config,\n", + " # This is the input file, a .nxs\n", + " \"input\": \"/dls/i12/data/2025/.......",\n", + " # This can be any folder on /dls/\n", + " \"output\": \"/dls/i12/data/2025/.......\",\n", + " \"recon_outdir_name\": \"sweep-run\", \"nprocs\": 1, \"memory\": \"1Gi\"\n", + " }\n", + " )\n", + " convert = convert_recon_data_format(\n", + " arguments={\n", + " \"recon_dir_path\": recon.get_parameter(\"out-path\"),\n", + " }\n", + " )\n", + " #run step recon before running step convert\n", + " recon >> convert # pyright: ignore\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3e196568", + "metadata": {}, + "outputs": [], + "source": [ + "#Firstly we write the yaml file we intend to send to GraphQL...\n", + "with open(\"/workspaces/{% endraw %}{{repo_name}}{% raw %}/src/{% endraw %}{{project_name}}{% raw %}/templates/visr_recon.txt\", \"w\") as div:\n", + " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31aa8293", + "metadata": {}, + "outputs": [], + "source": [ + "# Then we run this to submit the workflow to GraphQL, first linting the yaml.\n", + "from python_workflow_submitter.submit_workflow import submit_workflow_yaml\n", + "\n", + "await submit_workflow_yaml(\"/workspaces/{% endraw %}{{repo_name}}{% raw %}/src/{% endraw %}{{project_name}}{% raw %}/templates/realexample.yaml\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "imaging-python-workflows-interface-test (3.11.x)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.undefined" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} +{% endraw %} diff --git a/src/python_interface_to_workflows/templates/visr_recon.txt b/src/python_interface_to_workflows/templates/visr_recon.txt new file mode 100644 index 0000000..2630f7b --- /dev/null +++ b/src/python_interface_to_workflows/templates/visr_recon.txt @@ -0,0 +1,276 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: visr-recon-with-python-interface-to-workflows + annotations: + workflows.argoproj.io/description: |- + ViSR recon + example.yaml + workflows.argoproj.io/title: ViSR recon + workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows + labels: + workflows.diamond.ac.uk/science-group-imaging: 'true' +spec: + entrypoint: workflowentry + templates: + - name: workflowentry + dag: + tasks: + - name: tomo-recon + template: tomo-recon + arguments: + parameters: + - name: config + value: |- + [ + { + "method": "standard_tomo", + "module_path": "httomo.data.hdf.loaders", + "parameters": { + "data_path": "/entry1/tomo_entry/data/data", + "image_key_path": "/entry1/tomo_entry/instrument/detector/image_key", + "rotation_angles": { + "data_path": "/entry1/tomo_entry/data/rotation_angle" + }, + "preview": { + "detector_y": { + "start": 100, + "stop": 102 + } + } + } + }, + { + "method": "remove_outlier", + "module_path": "tomopy.misc.corr", + "parameters": { + "dif": 0.1, + "size": 3, + "axis": "auto" + } + }, + { + "method": "dark_flat_field_correction", + "module_path": "httomolibgpu.prep.normalize", + "parameters": { + "flats_multiplier": 1, + "darks_multiplier": 1 + } + }, + { + "method": "find_center_vo", + "module_path": "httomolibgpu.recon.rotation", + "parameters": { + "ind": null, + "smin": -50, + "smax": 50, + "srad": 6, + "step": 0.25, + "ratio": 0.5, + "drop": 20 + }, + "id": "centering", + "side_outputs": { + "cor": "centre_of_rotation" + } + }, + { + "method": "FBP3d_tomobar", + "module_path": "httomolibgpu.recon.algorithm", + "parameters": { + "center": "${{centering.side_outputs.centre_of_rotation}}", + "filter_freq_cutoff": 0.6, + "recon_size": null, + "recon_mask_radius": null + }, + "save_result": true + }, + { + "method": "calculate_stats", + "module_path": "httomo.methods", + "parameters": {}, + "id": "statistics", + "side_outputs": { + "glob_stats": "glob_stats" + } + }, + { + "method": "rescale_to_int", + "module_path": "httomolib.misc.rescale", + "parameters": { + "perc_range_min": 0, + "perc_range_max": 100, + "bits": 8, + "glob_stats": "${{statistics.side_outputs.glob_stats}}" + }, + "save_result": true + } + ] + - name: input + value: '/dls/i12/data/2025/.......' + - name: output + value: '/dls/i12/data/2025/.......' + - name: recon_outdir_name + value: sweep-run + - name: nprocs + value: '1' + - name: memory + value: 1Gi + - name: convert-recon-data-format + depends: tomo-recon + template: convert-recon-data-format + arguments: + parameters: + - name: recon_dir_path + value: '{{tasks.tomo-recon.outputs.parameters.out-path}}' + - name: tomo-recon + podSpecPatch: '{"containers": [{"name": "main", "resources": {"limits": {"cpu": + "{{inputs.parameters.nprocs}}", "memory": "{{inputs.parameters.memory}}", "nvidia.com/gpu": + "{{inputs.parameters.nprocs}}"}, "requests": {"cpu": "{{inputs.parameters.nprocs}}", + "memory": "{{inputs.parameters.memory}}", "nvidia.com/gpu": "{{inputs.parameters.nprocs}}"}}}]}' + tolerations: + - effect: NoSchedule + key: nvidia.com/gpu + operator: Exists + - effect: NoSchedule + key: nodetype + operator: Equal + value: gpu + - effect: NoSchedule + key: nodegroup + operator: Equal + value: workflows + inputs: + parameters: + - name: config + - name: input + - name: output + - name: recon_outdir_name + - name: nprocs + - name: memory + outputs: + parameters: + - name: out-path + valueFrom: + path: /tmp/parameters.json + script: + image: ghcr.io/diamondlightsource/httomo:latest + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: config = json.loads(r'''{{inputs.parameters.config}}''') + except: config = r'''{{inputs.parameters.config}}''' + try: input = json.loads(r'''{{inputs.parameters.input}}''') + except: input = r'''{{inputs.parameters.input}}''' + try: memory = json.loads(r'''{{inputs.parameters.memory}}''') + except: memory = r'''{{inputs.parameters.memory}}''' + try: nprocs = json.loads(r'''{{inputs.parameters.nprocs}}''') + except: nprocs = r'''{{inputs.parameters.nprocs}}''' + try: output = json.loads(r'''{{inputs.parameters.output}}''') + except: output = r'''{{inputs.parameters.output}}''' + try: recon_outdir_name = json.loads(r'''{{inputs.parameters.recon_outdir_name}}''') + except: recon_outdir_name = r'''{{inputs.parameters.recon_outdir_name}}''' + + import json + import subprocess + loaded_config = json.dumps(config) + subprocess.check_call(['/opt/conda/bin/mpirun', '-n', str(nprocs), '/opt/conda/bin/python', '-m', 'httomo', 'run', '--pipeline-format', 'json', '--output-folder-name', recon_outdir_name, input, loaded_config, output]) + with open('/tmp/parameters.json', 'w') as f: + json.dump(f'{output}/{recon_outdir_name}', f) + command: + - /opt/conda/bin/python + env: + - name: CUPY_CACHE_DIR + value: /tmp/.cupy/kernel_cache + - name: MKL_NUM_THREADS + value: '1' + - name: NUMEXPR_NUM_THREADS + value: '1' + - name: OMP_NUM_THREADS + value: '1' + volumeMounts: + - name: session + mountPath: '{{workflow.parameters.visitdir}}' + - name: tmpdir + mountPath: /tmp + - name: convert-recon-data-format + inputs: + parameters: + - name: recon_dir_path + - name: tmpdir_path + default: /tmp + - name: raw_recon_filename + default: recon.raw + - name: metadata_filename + default: metadata.json + outputs: + artifacts: + - name: recon + path: '{{inputs.parameters.tmpdir_path}}/{{inputs.parameters.raw_recon_filename}}' + archive: + none: {} + - name: metadata + path: '{{inputs.parameters.tmpdir_path}}/{{inputs.parameters.metadata_filename}}' + archive: + none: {} + script: + image: ghcr.io/diamondlightsource/httomo:latest + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: metadata_filename = json.loads(r'''{{inputs.parameters.metadata_filename}}''') + except: metadata_filename = r'''{{inputs.parameters.metadata_filename}}''' + try: raw_recon_filename = json.loads(r'''{{inputs.parameters.raw_recon_filename}}''') + except: raw_recon_filename = r'''{{inputs.parameters.raw_recon_filename}}''' + try: recon_dir_path = json.loads(r'''{{inputs.parameters.recon_dir_path}}''') + except: recon_dir_path = r'''{{inputs.parameters.recon_dir_path}}''' + try: tmpdir_path = json.loads(r'''{{inputs.parameters.tmpdir_path}}''') + except: tmpdir_path = r'''{{inputs.parameters.tmpdir_path}}''' + + import json + from pathlib import Path + import h5py + RAW_RECON_PATH = f'{tmpdir_path}/{raw_recon_filename}' + HDF5_RECON_DIR = Path(recon_dir_path) + HDF5_RECON_FILENAME_PATTERN = '*-httomolib-rescale_to_int.h5' + hdf5_recon_data_path = list(HDF5_RECON_DIR.glob(HDF5_RECON_FILENAME_PATTERN))[0] + with h5py.File(hdf5_recon_data_path, 'r') as f: + data = f['/data'][:] + data.tofile(RAW_RECON_PATH) + METADATA_PATH = f'{tmpdir_path}/{metadata_filename}' + order = 'C' if data.flags.c_contiguous else 'F' + metadata = {'shape': list(data.shape), 'dtype': str(data.dtype), 'order': order} + with open(METADATA_PATH, 'w') as f: + f.write(json.dumps(metadata, indent=2)) + command: + - /opt/conda/bin/python + volumeMounts: + - name: session + mountPath: '{{workflow.parameters.visitdir}}' + - name: tmpdir + mountPath: /tmp + volumeClaimTemplates: + - metadata: + name: tmpdir + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + volumes: + - name: session + hostPath: + path: '{{workflow.parameters.visitdir}}' + type: Directory + arguments: + parameters: + - name: visitdir + valueFrom: + configMapKeyRef: + name: sessionspaces + key: data_directory diff --git a/src/python_interface_to_workflows/workflow_definitions/visr_notebook_example/visr-recon.ipynb b/src/python_interface_to_workflows/workflow_definitions/visr_notebook_example/visr-recon.ipynb new file mode 100644 index 0000000..00703a9 --- /dev/null +++ b/src/python_interface_to_workflows/workflow_definitions/visr_notebook_example/visr-recon.ipynb @@ -0,0 +1,372 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "a4baf3e4", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import json\n", + "\n", + "from hera.shared import global_config\n", + "from hera.workflows import (\n", + " Artifact,\n", + " DAG,\n", + " Env,\n", + " Parameter,\n", + " Script,\n", + " Volume,\n", + " Workflow,\n", + " script, # pyright: ignore[reportUnknownVariableType]\n", + ")\n", + "from hera.workflows import models as m\n", + "from hera.workflows.archive import NoneArchiveStrategy\n", + "\n", + "# Sets the default image, unless specified otherwise, to this.\n", + "global_config.set_class_defaults( # pyright: ignore\n", + " Script, image=\"ghcr.io/diamondlightsource/httomo:latest\"\n", + ")\n", + "# Do note! This workflow will not run, as the DLS paths have been replaced with\n", + "# placeholders, so as not to fill up required directories." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "98316b4f", + "metadata": {}, + "outputs": [], + "source": [ + "# The script decorator allows hera to convert python code into yaml\n", + "@script(\n", + "# Assure that we only run on a pod with certain parameters\n", + " pod_spec_patch=json.dumps({\"containers\":\n", + " [{\"name\":\"main\",\n", + " \"resources\":\n", + " {\"limits\":{\"cpu\":\"{{inputs.parameters.nprocs}}\",\n", + " \"memory\":\"{{inputs.parameters.memory}}\",\n", + " \"nvidia.com/gpu\":\"{{inputs.parameters.nprocs}}\"},\n", + " \"requests\":{\"cpu\":\"{{inputs.parameters.nprocs}}\",\n", + " \"memory\":\"{{inputs.parameters.memory}}\",\n", + " \"nvidia.com/gpu\":\"{{inputs.parameters.nprocs}}\"}}}]}),\n", + "\n", + "# Assure that we only run on a pod with certain parameters\n", + " tolerations=[\n", + " m.Toleration(key=\"nvidia.com/gpu\",operator=\"Exists\",effect=\"NoSchedule\"),\n", + " m.Toleration(key=\"nodetype\",operator=\"Equal\",value=\"gpu\",effect=\"NoSchedule\"),\n", + " m.Toleration(key=\"nodegroup\",operator=\"Equal\",value=\"workflows\",effect=\"NoSchedule\")],\n", + "# Inform the pod which installation of python we intend to run on\n", + " command=[\"/opt/conda/bin/python\"],\n", + "# mount both volumes - one being a parameters in the diamond file system, the other\n", + "# being a mounted file path\n", + " volume_mounts=[\n", + " m.VolumeMount(name=\"session\", mount_path=\"{{workflow.parameters.visitdir}}\"),\n", + " m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\"),\n", + " ],\n", + "# inform the next step that the file we produce here will be used as a parameter for the\n", + "# next script\n", + " outputs=[\n", + " Parameter(\n", + " name=\"out-path\",\n", + " value_from=m.ValueFrom(\n", + " path=\"/tmp/parameters.json\"\n", + " )\n", + " )\n", + " ],\n", + "# establish some environmental variables shared between steps.\n", + " env=[\n", + " Env(name=\"CUPY_CACHE_DIR\", value=\"/tmp/.cupy/kernel_cache\"),\n", + " Env(name=\"MKL_NUM_THREADS\", value=\"1\"),\n", + " Env(name=\"NUMEXPR_NUM_THREADS\", value=\"1\"),\n", + " Env(name=\"OMP_NUM_THREADS\", value=\"1\"),\n", + " ]\n", + ")\n", + "def tomo_recon(\n", + " config: str, input: str, output: str, recon_outdir_name: str, nprocs: int, memory: str\n", + "):\n", + " import json\n", + " import subprocess\n", + "# load the config string as a json - otherwise, in the yaml conversion, this is loaded\n", + "# as a string instead, which prevents mpirun from functioning.\n", + " loaded_config = json.dumps(config)\n", + "\n", + " subprocess.check_call([\n", + " \"/opt/conda/bin/mpirun\",\n", + " \"-n\",\n", + " str(nprocs),\n", + " \"/opt/conda/bin/python\",\n", + " \"-m\",\n", + " \"httomo\",\n", + " \"run\",\n", + " \"--pipeline-format\",\n", + " \"json\",\n", + " \"--output-folder-name\",\n", + " recon_outdir_name,\n", + " input,\n", + " loaded_config,\n", + " output\n", + " ])\n", + "\n", + " with open(\"/tmp/parameters.json\", \"w\") as f:\n", + " json.dump(f\"{output}/{recon_outdir_name}\", f)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "1535543f", + "metadata": {}, + "outputs": [], + "source": [ + "@script(\n", + " command=[\"/opt/conda/bin/python\"],\n", + " volume_mounts=[\n", + " m.VolumeMount(name=\"session\", mount_path=\"{{workflow.parameters.visitdir}}\"),\n", + " m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\"),\n", + " ],\n", + " #our outputs this time are artifacts, that we do not want compressed.\n", + " outputs=[\n", + " Artifact(\n", + " name=\"recon\",\n", + " path=\"{{inputs.parameters.tmpdir_path}}/{{inputs.parameters.raw_recon_filename}}\",\n", + " archive=NoneArchiveStrategy(),\n", + " ),\n", + " Artifact(\n", + " name=\"metadata\",\n", + " path=\"{{inputs.parameters.tmpdir_path}}/{{inputs.parameters.metadata_filename}}\",\n", + " archive=NoneArchiveStrategy(),\n", + " ),\n", + " ]\n", + ")\n", + "def convert_recon_data_format(recon_dir_path: str,\n", + " tmpdir_path: str = \"/tmp\",\n", + " raw_recon_filename: str = \"recon.raw\",\n", + " metadata_filename: str = \"metadata.json\"):\n", + " import json\n", + " from pathlib import Path\n", + "\n", + " import h5py\n", + "\n", + "\n", + "\n", + " RAW_RECON_PATH = f\"{tmpdir_path}/{raw_recon_filename}\"\n", + " HDF5_RECON_DIR = Path(recon_dir_path)\n", + " HDF5_RECON_FILENAME_PATTERN = \"*-httomolib-rescale_to_int.h5\"\n", + " hdf5_recon_data_path = list(HDF5_RECON_DIR.glob(HDF5_RECON_FILENAME_PATTERN))[0]\n", + "\n", + " with h5py.File(hdf5_recon_data_path, \"r\") as f:\n", + " data = f[\"/data\"][:]\n", + " data.tofile(RAW_RECON_PATH)\n", + "\n", + " METADATA_PATH = f\"{tmpdir_path}/{metadata_filename}\"\n", + "\n", + " order = \"C\" if data.flags.c_contiguous else \"F\"\n", + " metadata = {\"shape\": list(data.shape), \"dtype\": str(data.dtype), \"order\": order}\n", + " with open(METADATA_PATH, \"w\") as f:\n", + " f.write(json.dumps(metadata, indent=2))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7fdf9da0", + "metadata": {}, + "outputs": [], + "source": [ + "from hera.workflows.volume import HostPathVolume\n", + "\n", + "with Workflow(\n", + " # name of workflow - will append a short identifier automatically.\n", + " name=\"visr-recon-with-python-interface-to-workflows\",\n", + " # the following is the same as for writing any yaml workflow, but as variables.\n", + " # All of these are required aside from \"workflows.argoproj.io/description\".\n", + " entrypoint=\"workflowentry\",\n", + " api_version=\"argoproj.io/v1alpha1\",\n", + " kind=\"WorkflowTemplate\",\n", + " labels={\"workflows.diamond.ac.uk/science-group-imaging\": \"true\"},\n", + " annotations={\n", + " \"workflows.argoproj.io/title\": \"ViSR recon\",\n", + " \"workflows.argoproj.io/description\": \"\"\"ViSR recon\n", + "example.yaml\"\"\",\n", + " \"workflows.diamond.ac.uk/repository\": \"https://github.com/DiamondLightSource/python-interface-to-workflows\",\n", + " },\n", + " # mounts the diamond file system\n", + " volumes=[\n", + " Volume(name=\"tmpdir\", mount_path=\"/tmp/\", size=\"1Gi\"),\n", + " HostPathVolume(name=\"session\",\n", + " path=\"{{workflow.parameters.visitdir}}\",\n", + " type=\"Directory\"),\n", + " ],\n", + " arguments=m.Arguments(\n", + " parameters=[\n", + " m.Parameter(\n", + " name=\"visitdir\",\n", + " value_from=m.ValueFrom(\n", + " config_map_key_ref=m.ConfigMapKeySelector(\n", + " name=\"sessionspaces\",\n", + " key=\"data_directory\"\n", + " )\n", + " )\n", + " )\n", + " ]\n", + " )\n", + ") as w:\n", + " with DAG(name=\"workflowentry\"):\n", + " # Alternatively, we could have this in /mounted_files/ and run\n", + " # a simple step where we copy over this json, before running\n", + " # the step in the httomo image\n", + " config = \"\"\"[\n", + " {\n", + " \"method\": \"standard_tomo\",\n", + " \"module_path\": \"httomo.data.hdf.loaders\",\n", + " \"parameters\": {\n", + " \"data_path\": \"/entry1/tomo_entry/data/data\",\n", + " \"image_key_path\": \"/entry1/tomo_entry/instrument/detector/image_key\",\n", + " \"rotation_angles\": {\n", + " \"data_path\": \"/entry1/tomo_entry/data/rotation_angle\"\n", + " },\n", + " \"preview\": {\n", + " \"detector_y\": {\n", + " \"start\": 100,\n", + " \"stop\": 102\n", + " }\n", + " }\n", + " }\n", + " },\n", + " {\n", + " \"method\": \"remove_outlier\",\n", + " \"module_path\": \"tomopy.misc.corr\",\n", + " \"parameters\": {\n", + " \"dif\": 0.1,\n", + " \"size\": 3,\n", + " \"axis\": \"auto\"\n", + " }\n", + " },\n", + " {\n", + " \"method\": \"dark_flat_field_correction\",\n", + " \"module_path\": \"httomolibgpu.prep.normalize\",\n", + " \"parameters\": {\n", + " \"flats_multiplier\": 1,\n", + " \"darks_multiplier\": 1\n", + " }\n", + " },\n", + " {\n", + " \"method\": \"find_center_vo\",\n", + " \"module_path\": \"httomolibgpu.recon.rotation\",\n", + " \"parameters\": {\n", + " \"ind\": null,\n", + " \"smin\": -50,\n", + " \"smax\": 50,\n", + " \"srad\": 6,\n", + " \"step\": 0.25,\n", + " \"ratio\": 0.5,\n", + " \"drop\": 20\n", + " },\n", + " \"id\": \"centering\",\n", + " \"side_outputs\": {\n", + " \"cor\": \"centre_of_rotation\"\n", + " }\n", + " },\n", + " {\n", + " \"method\": \"FBP3d_tomobar\",\n", + " \"module_path\": \"httomolibgpu.recon.algorithm\",\n", + " \"parameters\": {\n", + " \"center\": \"${{centering.side_outputs.centre_of_rotation}}\",\n", + " \"filter_freq_cutoff\": 0.6,\n", + " \"recon_size\": null,\n", + " \"recon_mask_radius\": null\n", + " },\n", + " \"save_result\": true\n", + " },\n", + " {\n", + " \"method\": \"calculate_stats\",\n", + " \"module_path\": \"httomo.methods\",\n", + " \"parameters\": {},\n", + " \"id\": \"statistics\",\n", + " \"side_outputs\": {\n", + " \"glob_stats\": \"glob_stats\"\n", + " }\n", + " },\n", + " {\n", + " \"method\": \"rescale_to_int\",\n", + " \"module_path\": \"httomolib.misc.rescale\",\n", + " \"parameters\": {\n", + " \"perc_range_min\": 0,\n", + " \"perc_range_max\": 100,\n", + " \"bits\": 8,\n", + " \"glob_stats\": \"${{statistics.side_outputs.glob_stats}}\"\n", + " },\n", + " \"save_result\": true\n", + " }\n", + "]\"\"\"\n", + " recon = tomo_recon(\n", + " arguments={\n", + " \"config\": config,\n", + " # This is the input file, a .nxs\n", + " \"input\": \"/dls/i12/data/2025/.......\",\n", + " # This can be any folder on /dls/ where we wish to put the file.\n", + " \"output\": \"/dls/i12/data/2025/........\",\n", + " \"recon_outdir_name\": \"sweep-run\", \"nprocs\": 1, \"memory\": \"1Gi\"\n", + " }\n", + " )\n", + " convert = convert_recon_data_format(\n", + " arguments={\n", + " \"recon_dir_path\": recon.get_parameter(\"out-path\"),\n", + " }\n", + " )\n", + " #run step recon before running step convert\n", + " recon >> convert # pyright: ignore\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "3e196568", + "metadata": {}, + "outputs": [], + "source": [ + "#Firstly we write the yaml file we intend to send to GraphQL...\n", + "with open(\"/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/templates/visr_recon.txt\", \"w\") as div:\n", + " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31aa8293", + "metadata": {}, + "outputs": [], + "source": [ + "# Then we run this to submit the workflow to GraphQL, first linting the yaml.\n", + "from python_workflow_submitter.submit_workflow import submit_workflow_yaml\n", + "\n", + "await submit_workflow_yaml(\"/workspaces/imaging-python-workflows/src/imaging_python_workflows/templates/visr_recon.yaml\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python-interface-to-workflows (3.11.x)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 442b5b3b0d778b9a5d058ae1d278914be5cc163b Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Mon, 17 Aug 2026 14:53:22 +0000 Subject: [PATCH 2/2] docs(copier): adds further comments explaining how to vary params within workflows --- .../create_notebook_in_image.py.jinja | 8 +++++++- .../workflow_definitions/create_notebook_in_image.py | 7 +++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja index d209786..e862091 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja @@ -52,7 +52,13 @@ def mount_files(): shell=True, ) - +# If you wanted to make something that submits multiple versions of the same workflow, +# simply copy the following section and write 'as a' instead of 'as w', +# then write "a.to_yaml" to a new file, etc. +# To run them all, as you have to call submit_workflow_yaml once per line, simply +# write out a submit_workflow_yaml once per file, or write a for loop to submit each +# sequentially. +# This way, you can alter input parameters you assign (see visr example) with Workflow( # assures that the container has enough resources for our workflow pod_spec_patch=json.dumps( diff --git a/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py b/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py index 5df63fd..9562091 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py @@ -52,6 +52,13 @@ def mount_files(): ) +# If you wanted to make something that submits multiple versions of the same workflow, +# simply copy the following section and write 'as a' instead of 'as w', +# then write "a.to_yaml" to a new file, etc. +# To run them all, as you have to call submit_workflow_yaml once per line, simply +# write out a submit_workflow_yaml once per file, or write a for loop to submit each +# sequentially. +# This way, you can alter input parameters you assign (see visr example) with Workflow( # assures that the container has enough resources for our workflow pod_spec_patch=json.dumps(