diff --git a/CHANGES.md b/CHANGES.md index 6b6327c1..e02a1368 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,9 @@ development source code and as such may not be routinely kept up to date. ## Bug fixes +* `nextstrain build --aws-batch` now waits 10 minutes before automatically + canceling any jobs that are stuck as RUNNABLE due to misconfiguration. + ([#551](https://github.com/nextstrain/cli/pull/551)) * Commands that authenticate with nextstrain.org, such as `nextstrain login`, `nextstrain whoami`, and `nextstrain remote` with nextstrain.org addresses, no longer load AWS credentials, which they don't use. Previously, a local diff --git a/doc/changes.md b/doc/changes.md index 01f61c73..959c9621 100644 --- a/doc/changes.md +++ b/doc/changes.md @@ -19,6 +19,9 @@ development source code and as such may not be routinely kept up to date. (v-next-bug-fixes)= ### Bug fixes +* `nextstrain build --aws-batch` now waits 10 minutes before automatically + canceling any jobs that are stuck as RUNNABLE due to misconfiguration. + ([#551](https://github.com/nextstrain/cli/pull/551)) * Commands that authenticate with nextstrain.org, such as `nextstrain login`, `nextstrain whoami`, and `nextstrain remote` with nextstrain.org addresses, no longer load AWS credentials, which they don't use. Previously, a local diff --git a/nextstrain/cli/runner/aws_batch/__init__.py b/nextstrain/cli/runner/aws_batch/__init__.py index 011cacb0..07c94456 100644 --- a/nextstrain/cli/runner/aws_batch/__init__.py +++ b/nextstrain/cli/runner/aws_batch/__init__.py @@ -420,11 +420,15 @@ def interrupt_signaled(sig, frame): if job.status_changed and not job.is_complete: print_stage("Job now %s" % job.status) - # Cancel jobs that are stuck due to misconfiguration. - # + # Cancel jobs that are stuck due to misconfiguration.¹ The 10 minute + # timeout is buffer time for EC2 instance scale-up, etc. and mirrors the + # minimum duration of the AWS-side equivalent.² + # ¹ + # ² if (job.status == "RUNNABLE" and job.status_reason is not None and job.status_reason.startswith("MISCONFIGURATION") + and job.elapsed_time >= 600 and not job.stop_sent): warn(f"Job is stuck in RUNNABLE. Reason: {job.status_reason}") print_stage("Automatically canceling stuck job…") diff --git a/nextstrain/cli/runner/aws_batch/jobs.py b/nextstrain/cli/runner/aws_batch/jobs.py index 9901d6e6..97bd8c6a 100644 --- a/nextstrain/cli/runner/aws_batch/jobs.py +++ b/nextstrain/cli/runner/aws_batch/jobs.py @@ -95,9 +95,9 @@ def is_failed(self) -> bool: @property def elapsed_time(self) -> float: # Timestamps from the job state are in milliseconds - created = self.state.get("createdAt", float("NaN")) / 1000 - stopped = self.state.get("stoppedAt", time() * 1000) / 1000 - return stopped - created + start = self.state.get("createdAt", float("NaN")) / 1000 + end = self.state.get("stoppedAt", time() * 1000) / 1000 + return end - start @property def log_stream(self) -> Optional[str]: