diff --git a/CHANGES.md b/CHANGES.md index 61fcaed3..130d6104 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,16 @@ development source code and as such may not be routinely kept up to date. # __NEXT__ +## Improvements + +* `nextstrain build --aws-batch` now automatically cancels any jobs that are + stuck as RUNNABLE due to misconfiguration, while attached to the job. If the + job was stuck due to the requested resources, a helpful message is shown. + ([#547](https://github.com/nextstrain/cli/pull/547)) +* Documentation for AWS Batch setup has been updated to recommend setting job + state limits on the job queue. + ([#547](https://github.com/nextstrain/cli/pull/547)) + ## Bug fixes * `nextstrain build --aws-batch` now handles network connection errors diff --git a/doc/aws-batch.md b/doc/aws-batch.md index ceef5fa6..454027c2 100644 --- a/doc/aws-batch.md +++ b/doc/aws-batch.md @@ -319,6 +319,9 @@ the `NEXTSTRAIN_AWS_BATCH_QUEUE` environment variable, or set `queue` in the If you're not using the wizard, make sure you connect the job queue to the compute environment you created above. +While not required, you can add _[Job state limits]_ for a better user experience. + +[Job state limits]: https://docs.aws.amazon.com/batch/latest/userguide/job_stuck_in_runnable_time_limit_actions.html #### Job definition diff --git a/doc/changes.md b/doc/changes.md index b1b47c30..e36c050e 100644 --- a/doc/changes.md +++ b/doc/changes.md @@ -16,6 +16,17 @@ development source code and as such may not be routinely kept up to date. (v-next)= ## __NEXT__ +(v-next-improvements)= +### Improvements + +* `nextstrain build --aws-batch` now automatically cancels any jobs that are + stuck as RUNNABLE due to misconfiguration, while attached to the job. If the + job was stuck due to the requested resources, a helpful message is shown. + ([#547](https://github.com/nextstrain/cli/pull/547)) +* Documentation for AWS Batch setup has been updated to recommend setting job + state limits on the job queue. + ([#547](https://github.com/nextstrain/cli/pull/547)) + (v-next-bug-fixes)= ### Bug fixes diff --git a/nextstrain/cli/runner/aws_batch/__init__.py b/nextstrain/cli/runner/aws_batch/__init__.py index 91e578e3..011cacb0 100644 --- a/nextstrain/cli/runner/aws_batch/__init__.py +++ b/nextstrain/cli/runner/aws_batch/__init__.py @@ -420,6 +420,16 @@ def interrupt_signaled(sig, frame): if job.status_changed and not job.is_complete: print_stage("Job now %s" % job.status) + # Cancel jobs that are stuck due to misconfiguration. + # + if (job.status == "RUNNABLE" + and job.status_reason is not None + and job.status_reason.startswith("MISCONFIGURATION") + and not job.stop_sent): + warn(f"Job is stuck in RUNNABLE. Reason: {job.status_reason}") + print_stage("Automatically canceling stuck job…") + job.stop(job.status_reason) + if job.is_running and not log_watcher: # Transitioned from waiting → running, so kick off the log watcher. if opts.logs: @@ -444,6 +454,15 @@ def interrupt_signaled(sig, frame): print_stage( "Job %s after %0.1f minutes" % (job.status, job.elapsed_time / 60), "(%s)" % job.status_reason) + + if (job.status == "FAILED" + and job.status_reason is not None + and "MISCONFIGURATION:JOB_RESOURCE_REQUIREMENT" in job.status_reason): + warn(dedent(f""" + Requested resources of {job.cpus} CPUs and/or {job.memory} MiB memory are likely beyond the limits of the largest instance size allowed by the job queue. + + Try adjusting your requested resources or the job queue in AWS.""")) + break # Only check status every 6s (10 times per minute). @@ -511,7 +530,7 @@ def interrupt(job: jobs.JobState, confirm: bool = stdin.isatty()) -> bool: print_stage("Send SIGINT again within %d seconds to cancel this job." % (CTRL_C_CONFIRMATION_TIMEOUT,)) else: print_stage("Canceling job…") - job.stop() + job.stop("stopped by user") print_stage("Waiting for job to stop…") if stdin.isatty(): diff --git a/nextstrain/cli/runner/aws_batch/jobs.py b/nextstrain/cli/runner/aws_batch/jobs.py index 217780ea..9901d6e6 100644 --- a/nextstrain/cli/runner/aws_batch/jobs.py +++ b/nextstrain/cli/runner/aws_batch/jobs.py @@ -26,7 +26,9 @@ class JobState: RUNNING_STATUS = { 'RUNNING' } TERMINAL_STATUS = { 'SUCCEEDED', 'FAILED' } - STOP_REASON = "stopped by user" + # Formatted to match the status reason set by jobStateTimeLimitActions: + # "Canceled by JobStateTimeLimit action due to reason: MISCONFIGURATION:JOB_RESOURCE_REQUIREMENT" + STOP_REASON_PREFIX = "Canceled by Nextstrain CLI due to reason: " def __init__(self, job_id): self.id = job_id @@ -115,6 +117,20 @@ def workdir(self) -> Optional[s3.S3Object]: return s3.object_from_url(url) if url else None + @property + def cpus(self) -> Optional[int]: + for req in self.state.get("container", {}).get("resourceRequirements", []): + if req.get("type") == "VCPU": + return int(req["value"]) + return None + + @property + def memory(self) -> Optional[int]: + for req in self.state.get("container", {}).get("resourceRequirements", []): + if req.get("type") == "MEMORY": + return int(req["value"]) + return None + def log_entries(self) -> Generator[dict, None, None]: """ Fetch all the CloudWatch log entries for this job. @@ -137,16 +153,18 @@ def log_watcher(self, consumer: Callable[[dict], None]) -> logs.LogWatcher: return logs.LogWatcher(self.log_stream, consumer) - def stop(self) -> None: + def stop(self, reason: str) -> None: """ Stop the job, regardless of if it has started yet or not. """ - self._client.terminate_job(jobId = self.id, reason = self.STOP_REASON) + full_reason = self.STOP_REASON_PREFIX + reason + self._client.terminate_job(jobId = self.id, reason = full_reason) self.stop_sent = True @property def stopped(self) -> bool: - return self.status_reason == self.STOP_REASON + # NOTE: This does not capture external stop reasons (e.g. set by AWS). + return bool(self.status_reason and self.status_reason.startswith(self.STOP_REASON_PREFIX)) def submit(name: str,