diff --git a/ansible/roles/compiled/templates/cromwell_slurm.conf b/ansible/roles/compiled/templates/cromwell_slurm.conf index 264f321..6aebdbd 100644 --- a/ansible/roles/compiled/templates/cromwell_slurm.conf +++ b/ansible/roles/compiled/templates/cromwell_slurm.conf @@ -80,10 +80,15 @@ backend { # Submit the script to SLURM. `singularity` here and above comes from the # unversioned `singularity` module that cromwell.lua loads; sbatch exports # the submitting environment into the job, so no path is hard-coded. - # No wait flag (unlike the upstream docs example): the submit returns at once - # so Cromwell learns the job id, can scancel it on abort, and detects - # completion through check-alive and the rc file exactly as `submit` does. + # The wait flag, as in the upstream docs example, blocks until the job ends + # and returns its exit status, so a job SLURM kills (time, memory) fails the + # task at once instead of waiting forever for an rc file that never comes. + # NOTE: this costs abort (no job id until the job ends, so kill-docker cannot + # scancel it) and holds a backend thread per running docker task. It was + # removed for those reasons and restored to diagnose a user's failures; + # exit-code-timeout-seconds is the non-blocking alternative to evaluate. sbatch \ + --wait \ -J ${job_name} \ -D ${cwd} \ -o ${cwd}/execution/stdout \