Auto-cancel jobs stuck as RUNNABLE

CHANGES.md

@@ -13,6 +13,16 @@ development source code and as such may not be routinely kept up to date.
 
 # __NEXT__
 
+## Improvements
+
+* `nextstrain build --aws-batch` now automatically cancels any jobs that are
+  stuck as RUNNABLE due to misconfiguration, while attached to the job. If the
+  job was stuck due to the requested resources, a helpful message is shown.
+  ([#547](https://github.com/nextstrain/cli/pull/547))
+* Documentation for AWS Batch setup has been updated to recommend setting job
+  state limits on the job queue.
+  ([#547](https://github.com/nextstrain/cli/pull/547))
+
 ## Bug fixes
 
 * `nextstrain build --aws-batch` now handles network connection errors

doc/aws-batch.md

@@ -319,6 +319,9 @@ the `NEXTSTRAIN_AWS_BATCH_QUEUE` environment variable, or set `queue` in the
 If you're not using the wizard, make sure you connect the job queue to the
 compute environment you created above.
 
+While not required, you can add _[Job state limits]_ for a better user experience.
+
+[Job state limits]: https://docs.aws.amazon.com/batch/latest/userguide/job_stuck_in_runnable_time_limit_actions.html
 
 #### Job definition
 

doc/changes.md

@@ -16,6 +16,17 @@ development source code and as such may not be routinely kept up to date.
 (v-next)=
 ## __NEXT__
 
+(v-next-improvements)=
+### Improvements
+
+* `nextstrain build --aws-batch` now automatically cancels any jobs that are
+  stuck as RUNNABLE due to misconfiguration, while attached to the job. If the
+  job was stuck due to the requested resources, a helpful message is shown.
+  ([#547](https://github.com/nextstrain/cli/pull/547))
+* Documentation for AWS Batch setup has been updated to recommend setting job
+  state limits on the job queue.
+  ([#547](https://github.com/nextstrain/cli/pull/547))
+
 (v-next-bug-fixes)=
 ### Bug fixes
 

nextstrain/cli/runner/aws_batch/__init__.py

@@ -420,6 +420,16 @@ def interrupt_signaled(sig, frame):
         if job.status_changed and not job.is_complete:
             print_stage("Job now %s" % job.status)
 
+        # Cancel jobs that are stuck due to misconfiguration.
+        # <https://docs.aws.amazon.com/batch/latest/userguide/job_stuck_in_runnable_misconfiguration.html>
+        if (job.status == "RUNNABLE"
+        and job.status_reason is not None
+        and job.status_reason.startswith("MISCONFIGURATION")
+        and not job.stop_sent):
+            warn(f"Job is stuck in RUNNABLE. Reason: {job.status_reason}")
+            print_stage("Automatically canceling stuck job…")
+            job.stop(job.status_reason)
+
         if job.is_running and not log_watcher:
             # Transitioned from waiting → running, so kick off the log watcher.
             if opts.logs:
@@ -444,6 +454,15 @@ def interrupt_signaled(sig, frame):
             print_stage(
                 "Job %s after %0.1f minutes" % (job.status, job.elapsed_time / 60),
                 "(%s)" % job.status_reason)
+
+            if (job.status == "FAILED"
+            and job.status_reason is not None
+            and "MISCONFIGURATION:JOB_RESOURCE_REQUIREMENT" in job.status_reason):
+                warn(dedent(f"""
+                    Requested resources of {job.cpus} CPUs and/or {job.memory} MiB memory are likely beyond the limits of the largest instance size allowed by the job queue.
+
+                    Try adjusting your requested resources or the job queue in AWS."""))
+
             break
 
         # Only check status every 6s (10 times per minute).
@@ -511,7 +530,7 @@ def interrupt(job: jobs.JobState, confirm: bool = stdin.isatty()) -> bool:
             print_stage("Send SIGINT again within %d seconds to cancel this job." % (CTRL_C_CONFIRMATION_TIMEOUT,))
     else:
         print_stage("Canceling job…")
-        job.stop()
+        job.stop("stopped by user")
 
         print_stage("Waiting for job to stop…")
         if stdin.isatty():

nextstrain/cli/runner/aws_batch/jobs.py

@@ -26,7 +26,9 @@ class JobState:
     RUNNING_STATUS  = { 'RUNNING' }
     TERMINAL_STATUS = { 'SUCCEEDED', 'FAILED' }
 
-    STOP_REASON = "stopped by user"
+    # Formatted to match the status reason set by jobStateTimeLimitActions:
+    #                    "Canceled by JobStateTimeLimit action due to reason: MISCONFIGURATION:JOB_RESOURCE_REQUIREMENT"
+    STOP_REASON_PREFIX = "Canceled by Nextstrain CLI due to reason: "
 
     def __init__(self, job_id):
         self.id              = job_id
@@ -115,6 +117,20 @@ def workdir(self) -> Optional[s3.S3Object]:
 
         return s3.object_from_url(url) if url else None
 
+    @property
+    def cpus(self) -> Optional[int]:
+        for req in self.state.get("container", {}).get("resourceRequirements", []):
+            if req.get("type") == "VCPU":
+                return int(req["value"])
+        return None
+
+    @property
+    def memory(self) -> Optional[int]:
+        for req in self.state.get("container", {}).get("resourceRequirements", []):
+            if req.get("type") == "MEMORY":
+                return int(req["value"])
+        return None
+
     def log_entries(self) -> Generator[dict, None, None]:
         """
         Fetch all the CloudWatch log entries for this job.
@@ -137,16 +153,18 @@ def log_watcher(self, consumer: Callable[[dict], None]) -> logs.LogWatcher:
 
         return logs.LogWatcher(self.log_stream, consumer)
 
-    def stop(self) -> None:
+    def stop(self, reason: str) -> None:
         """
         Stop the job, regardless of if it has started yet or not.
         """
-        self._client.terminate_job(jobId = self.id, reason = self.STOP_REASON)
+        full_reason = self.STOP_REASON_PREFIX + reason
+        self._client.terminate_job(jobId = self.id, reason = full_reason)
         self.stop_sent = True
 
     @property
     def stopped(self) -> bool:
-        return self.status_reason == self.STOP_REASON
+        # NOTE: This does not capture external stop reasons (e.g. set by AWS).
+        return bool(self.status_reason and self.status_reason.startswith(self.STOP_REASON_PREFIX))
 
 
 def submit(name: str,