From aea4841d9f49ddd5901934fada3a038797deadc3 Mon Sep 17 00:00:00 2001 From: Victor Lin Date: Fri, 9 Oct 2026 15:21:59 -0700 Subject: [PATCH 1/3] Recommend setting job state limits on job queue This prevents jobs from being stuck as RUNNABLE indefinitely due to misconfiguration. --- CHANGES.md | 6 ++++++ doc/aws-batch.md | 3 +++ doc/changes.md | 7 +++++++ 3 files changed, 16 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 61fcaed3..b3cbd59c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,12 @@ development source code and as such may not be routinely kept up to date. # __NEXT__ +## Improvements + +* Documentation for AWS Batch setup has been updated to recommend setting job + state limits on the job queue. + ([#547](https://github.com/nextstrain/cli/pull/547)) + ## Bug fixes * `nextstrain build --aws-batch` now handles network connection errors diff --git a/doc/aws-batch.md b/doc/aws-batch.md index ceef5fa6..454027c2 100644 --- a/doc/aws-batch.md +++ b/doc/aws-batch.md @@ -319,6 +319,9 @@ the `NEXTSTRAIN_AWS_BATCH_QUEUE` environment variable, or set `queue` in the If you're not using the wizard, make sure you connect the job queue to the compute environment you created above. +While not required, you can add _[Job state limits]_ for a better user experience. + +[Job state limits]: https://docs.aws.amazon.com/batch/latest/userguide/job_stuck_in_runnable_time_limit_actions.html #### Job definition diff --git a/doc/changes.md b/doc/changes.md index b1b47c30..48bff63a 100644 --- a/doc/changes.md +++ b/doc/changes.md @@ -16,6 +16,13 @@ development source code and as such may not be routinely kept up to date. (v-next)= ## __NEXT__ +(v-next-improvements)= +### Improvements + +* Documentation for AWS Batch setup has been updated to recommend setting job + state limits on the job queue. + ([#547](https://github.com/nextstrain/cli/pull/547)) + (v-next-bug-fixes)= ### Bug fixes From 3fa74475865ade3b6cec053072c8e90dc54fb701 Mon Sep 17 00:00:00 2001 From: Victor Lin Date: Wed, 14 Oct 2026 16:02:58 -0700 Subject: [PATCH 2/3] Auto-cancel jobs stuck due to misconfiguration This is useful both in absence of and in addition to any job state limits set on the job queue. Stopped job handling now considers auto-canceled jobs in addition to user-sent interrupts. --- CHANGES.md | 2 ++ doc/changes.md | 2 ++ nextstrain/cli/runner/aws_batch/__init__.py | 12 +++++++++++- nextstrain/cli/runner/aws_batch/jobs.py | 12 ++++++++---- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b3cbd59c..c54433e3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,8 @@ development source code and as such may not be routinely kept up to date. ## Improvements +* `nextstrain build --aws-batch` now automatically cancels any jobs that are + stuck as RUNNABLE due to misconfiguration, while attached to the job. * Documentation for AWS Batch setup has been updated to recommend setting job state limits on the job queue. ([#547](https://github.com/nextstrain/cli/pull/547)) diff --git a/doc/changes.md b/doc/changes.md index 48bff63a..25ecda11 100644 --- a/doc/changes.md +++ b/doc/changes.md @@ -19,6 +19,8 @@ development source code and as such may not be routinely kept up to date. (v-next-improvements)= ### Improvements +* `nextstrain build --aws-batch` now automatically cancels any jobs that are + stuck as RUNNABLE due to misconfiguration, while attached to the job. * Documentation for AWS Batch setup has been updated to recommend setting job state limits on the job queue. ([#547](https://github.com/nextstrain/cli/pull/547)) diff --git a/nextstrain/cli/runner/aws_batch/__init__.py b/nextstrain/cli/runner/aws_batch/__init__.py index 91e578e3..402dbdd2 100644 --- a/nextstrain/cli/runner/aws_batch/__init__.py +++ b/nextstrain/cli/runner/aws_batch/__init__.py @@ -420,6 +420,16 @@ def interrupt_signaled(sig, frame): if job.status_changed and not job.is_complete: print_stage("Job now %s" % job.status) + # Cancel jobs that are stuck due to misconfiguration. + # + if (job.status == "RUNNABLE" + and job.status_reason is not None + and job.status_reason.startswith("MISCONFIGURATION") + and not job.stop_sent): + warn(f"Job is stuck in RUNNABLE. Reason: {job.status_reason}") + print_stage("Automatically canceling stuck job…") + job.stop(job.status_reason) + if job.is_running and not log_watcher: # Transitioned from waiting → running, so kick off the log watcher. if opts.logs: @@ -511,7 +521,7 @@ def interrupt(job: jobs.JobState, confirm: bool = stdin.isatty()) -> bool: print_stage("Send SIGINT again within %d seconds to cancel this job." % (CTRL_C_CONFIRMATION_TIMEOUT,)) else: print_stage("Canceling job…") - job.stop() + job.stop("stopped by user") print_stage("Waiting for job to stop…") if stdin.isatty(): diff --git a/nextstrain/cli/runner/aws_batch/jobs.py b/nextstrain/cli/runner/aws_batch/jobs.py index 217780ea..a9fb808f 100644 --- a/nextstrain/cli/runner/aws_batch/jobs.py +++ b/nextstrain/cli/runner/aws_batch/jobs.py @@ -26,7 +26,9 @@ class JobState: RUNNING_STATUS = { 'RUNNING' } TERMINAL_STATUS = { 'SUCCEEDED', 'FAILED' } - STOP_REASON = "stopped by user" + # Formatted to match the status reason set by jobStateTimeLimitActions: + # "Canceled by JobStateTimeLimit action due to reason: MISCONFIGURATION:JOB_RESOURCE_REQUIREMENT" + STOP_REASON_PREFIX = "Canceled by Nextstrain CLI due to reason: " def __init__(self, job_id): self.id = job_id @@ -137,16 +139,18 @@ def log_watcher(self, consumer: Callable[[dict], None]) -> logs.LogWatcher: return logs.LogWatcher(self.log_stream, consumer) - def stop(self) -> None: + def stop(self, reason: str) -> None: """ Stop the job, regardless of if it has started yet or not. """ - self._client.terminate_job(jobId = self.id, reason = self.STOP_REASON) + full_reason = self.STOP_REASON_PREFIX + reason + self._client.terminate_job(jobId = self.id, reason = full_reason) self.stop_sent = True @property def stopped(self) -> bool: - return self.status_reason == self.STOP_REASON + # NOTE: This does not capture external stop reasons (e.g. set by AWS). + return bool(self.status_reason and self.status_reason.startswith(self.STOP_REASON_PREFIX)) def submit(name: str, From 4729e3f55554034ba28b8c626fe83cfd28a451c0 Mon Sep 17 00:00:00 2001 From: Victor Lin Date: Fri, 9 Oct 2026 15:04:24 -0700 Subject: [PATCH 3/3] Add a helpful message for MISCONFIGURATION:JOB_RESOURCE_REQUIREMENT Even with auto-canceling mitigations in place, the failed status reason can be cryptic to users. Provide more context and a hint to next steps. --- CHANGES.md | 4 +++- doc/changes.md | 4 +++- nextstrain/cli/runner/aws_batch/__init__.py | 9 +++++++++ nextstrain/cli/runner/aws_batch/jobs.py | 14 ++++++++++++++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c54433e3..130d6104 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,7 +16,9 @@ development source code and as such may not be routinely kept up to date. ## Improvements * `nextstrain build --aws-batch` now automatically cancels any jobs that are - stuck as RUNNABLE due to misconfiguration, while attached to the job. + stuck as RUNNABLE due to misconfiguration, while attached to the job. If the + job was stuck due to the requested resources, a helpful message is shown. + ([#547](https://github.com/nextstrain/cli/pull/547)) * Documentation for AWS Batch setup has been updated to recommend setting job state limits on the job queue. ([#547](https://github.com/nextstrain/cli/pull/547)) diff --git a/doc/changes.md b/doc/changes.md index 25ecda11..e36c050e 100644 --- a/doc/changes.md +++ b/doc/changes.md @@ -20,7 +20,9 @@ development source code and as such may not be routinely kept up to date. ### Improvements * `nextstrain build --aws-batch` now automatically cancels any jobs that are - stuck as RUNNABLE due to misconfiguration, while attached to the job. + stuck as RUNNABLE due to misconfiguration, while attached to the job. If the + job was stuck due to the requested resources, a helpful message is shown. + ([#547](https://github.com/nextstrain/cli/pull/547)) * Documentation for AWS Batch setup has been updated to recommend setting job state limits on the job queue. ([#547](https://github.com/nextstrain/cli/pull/547)) diff --git a/nextstrain/cli/runner/aws_batch/__init__.py b/nextstrain/cli/runner/aws_batch/__init__.py index 402dbdd2..011cacb0 100644 --- a/nextstrain/cli/runner/aws_batch/__init__.py +++ b/nextstrain/cli/runner/aws_batch/__init__.py @@ -454,6 +454,15 @@ def interrupt_signaled(sig, frame): print_stage( "Job %s after %0.1f minutes" % (job.status, job.elapsed_time / 60), "(%s)" % job.status_reason) + + if (job.status == "FAILED" + and job.status_reason is not None + and "MISCONFIGURATION:JOB_RESOURCE_REQUIREMENT" in job.status_reason): + warn(dedent(f""" + Requested resources of {job.cpus} CPUs and/or {job.memory} MiB memory are likely beyond the limits of the largest instance size allowed by the job queue. + + Try adjusting your requested resources or the job queue in AWS.""")) + break # Only check status every 6s (10 times per minute). diff --git a/nextstrain/cli/runner/aws_batch/jobs.py b/nextstrain/cli/runner/aws_batch/jobs.py index a9fb808f..9901d6e6 100644 --- a/nextstrain/cli/runner/aws_batch/jobs.py +++ b/nextstrain/cli/runner/aws_batch/jobs.py @@ -117,6 +117,20 @@ def workdir(self) -> Optional[s3.S3Object]: return s3.object_from_url(url) if url else None + @property + def cpus(self) -> Optional[int]: + for req in self.state.get("container", {}).get("resourceRequirements", []): + if req.get("type") == "VCPU": + return int(req["value"]) + return None + + @property + def memory(self) -> Optional[int]: + for req in self.state.get("container", {}).get("resourceRequirements", []): + if req.get("type") == "MEMORY": + return int(req["value"]) + return None + def log_entries(self) -> Generator[dict, None, None]: """ Fetch all the CloudWatch log entries for this job.