From eda8cb38922424eb7bc8b6f46142d7da495e1604 Mon Sep 17 00:00:00 2001 From: Satyam Trivedi Date: Fri, 16 Oct 2026 00:09:05 +0530 Subject: [PATCH 1/4] fix(aws-batch): add job lifecycle, scheduling policies, job definition type check and job state-change events Add SubmitJob, DescribeJobs, ListJobs, CancelJob and TerminateJob. Jobs move SUBMITTED -> PENDING -> RUNNABLE -> STARTING -> RUNNING -> SUCCEEDED or FAILED on the provider clock, honour dependsOn and array jobs, and follow the documented cancel vs terminate rules. SubmitJob validates the queue (exists, ENABLED), the job definition revision, dependencies, share identifier, array size, priority, timeout and tags. ListJobs supports the documented filters and pages with a stable cursor. Add scheduling-policy CRUD and validate schedulingPolicyArn on CreateJobQueue/UpdateJobQueue. Enforce the RegisterJobDefinition type enum (container, multinode) and the properties each type requires. Jobs and scheduling policies are taggable, and tags follow the TagResource limits and the reserved aws: prefix. Publish "Batch Job State Change" events to the EventBridge default bus from the shared serve scheduler (config.Tickable) and on job API calls. New state is included in snapshots. Part of #1496 --- internal/settle/settle.go | 5 + providers/aws/aws.go | 3 +- providers/aws/batch/batch.go | 44 +- providers/aws/batch/compute_environment.go | 4 + providers/aws/batch/events.go | 231 +++++ providers/aws/batch/job_definition.go | 33 +- providers/aws/batch/job_lifecycle.go | 529 ++++++++++++ providers/aws/batch/job_queue.go | 56 +- providers/aws/batch/jobs.go | 941 +++++++++++++++++++++ providers/aws/batch/jobs_paging_test.go | 238 ++++++ providers/aws/batch/jobs_test.go | 726 ++++++++++++++++ providers/aws/batch/scheduling_policy.go | 281 ++++++ providers/aws/batch/snapshot.go | 21 +- providers/aws/batch/tags.go | 128 ++- providers/aws/tickables_test.go | 14 +- server/aws/batch/handler.go | 70 +- server/aws/batch/jobs.go | 119 +++ server/aws/batch/jobs_paging_sdk_test.go | 96 +++ server/aws/batch/jobs_sdk_test.go | 240 ++++++ server/aws/batch/scheduling_policy.go | 104 +++ server/aws/batch/types_jobs.go | 326 +++++++ services/batch/driver/driver.go | 21 +- services/batch/driver/types.go | 152 ++++ 23 files changed, 4328 insertions(+), 54 deletions(-) create mode 100644 providers/aws/batch/events.go create mode 100644 providers/aws/batch/job_lifecycle.go create mode 100644 providers/aws/batch/jobs.go create mode 100644 providers/aws/batch/jobs_paging_test.go create mode 100644 providers/aws/batch/jobs_test.go create mode 100644 providers/aws/batch/scheduling_policy.go create mode 100644 server/aws/batch/jobs.go create mode 100644 server/aws/batch/jobs_paging_sdk_test.go create mode 100644 server/aws/batch/jobs_sdk_test.go create mode 100644 server/aws/batch/scheduling_policy.go create mode 100644 server/aws/batch/types_jobs.go diff --git a/internal/settle/settle.go b/internal/settle/settle.go index 6a1c6c9d2..3a7ba54df 100644 --- a/internal/settle/settle.go +++ b/internal/settle/settle.go @@ -83,6 +83,11 @@ const ( DefaultRestoreExpeditedSettle = 1 * time.Second DefaultRestoreStandardSettle = 3 * time.Second DefaultRestoreBulkSettle = 5 * time.Second + + // DefaultBatchJobPhase is how long an AWS Batch job dwells in each + // non-terminal phase (SUBMITTED, RUNNABLE, STARTING, RUNNING) before moving + // on, so a dependency-free job reaches SUCCEEDED 4 phases after submission. + DefaultBatchJobPhase = 1 * time.Second ) // Window is a read-time overlay describing a resource still settling into its diff --git a/providers/aws/aws.go b/providers/aws/aws.go index 8b848a975..5aafa2483 100644 --- a/providers/aws/aws.go +++ b/providers/aws/aws.go @@ -598,6 +598,7 @@ func wireLifecycleEvents(p *Provider) { p.Glue.SetEventPublisher(p.EventBridge) p.CloudWatch.SetEventPublisher(p.EventBridge) p.SageMaker.SetEventPublisher(p.EventBridge) + p.Batch.SetEventPublisher(p.EventBridge) } // awsDrivers assembles the resource-discovery driver set from the provider's @@ -639,7 +640,7 @@ func awsDrivers(p *Provider) *resourcediscovery.Drivers { func (p *Provider) Tickables() []config.Tickable { // SQS and EFS run before CloudWatch so the alarms evaluated in the same // tick see the queue gauges and AWS/EFS metrics they just published. - return []config.Tickable{p.SQS, p.EFS, p.CloudWatch, p.SSM, p.CloudFormation} + return []config.Tickable{p.SQS, p.EFS, p.CloudWatch, p.SSM, p.CloudFormation, p.Batch} } // Close tears down any real engines wired into the provider via diff --git a/providers/aws/batch/batch.go b/providers/aws/batch/batch.go index 85239d10d..d859144cf 100644 --- a/providers/aws/batch/batch.go +++ b/providers/aws/batch/batch.go @@ -1,7 +1,10 @@ // Package batch provides an in-memory mock implementation of AWS Batch: the -// compute-environment, job-queue, and job-definition control plane. Resources -// provision synchronously (VALID/ACTIVE immediately), ARNs and ECS cluster ARNs -// are synthesized, and job-definition revisions increment on re-register. +// compute-environment, job-queue, job-definition and scheduling-policy control +// plane plus the job lifecycle. Resources provision synchronously +// (VALID/ACTIVE immediately), ARNs and ECS cluster ARNs are synthesized, and +// job-definition revisions increment on re-register. Jobs run no workload: a +// submitted job walks SUBMITTED -> RUNNABLE -> STARTING -> RUNNING -> +// SUCCEEDED on the configured clock (instantly unless AsyncSettle is on). package batch import ( @@ -11,6 +14,7 @@ import ( "sync" "github.com/stackshy/cloudemu/v2/config" + "github.com/stackshy/cloudemu/v2/internal/awsevents" "github.com/stackshy/cloudemu/v2/internal/idgen" "github.com/stackshy/cloudemu/v2/internal/memstore" "github.com/stackshy/cloudemu/v2/services/batch/driver" @@ -24,6 +28,8 @@ const ( kindComputeEnvironment = "compute-environment" kindJobQueue = "job-queue" kindJobDefinition = "job-definition" + kindSchedulingPolicy = "scheduling-policy" + kindJob = "job" ) // Mock is an in-memory implementation of AWS Batch. Each resource kind lives in @@ -33,26 +39,40 @@ const ( // reads are copy-on-write (mutations build fresh values, never mutating stored // ones in place). type Mock struct { - computeEnvs *memstore.Store[driver.ComputeEnvironment] - jobQueues *memstore.Store[driver.JobQueue] - jobDefs *memstore.Store[driver.JobDefinition] - jobDefMaxRev *memstore.Store[int32] + computeEnvs *memstore.Store[driver.ComputeEnvironment] + jobQueues *memstore.Store[driver.JobQueue] + jobDefs *memstore.Store[driver.JobDefinition] + jobDefMaxRev *memstore.Store[int32] + schedPolicies *memstore.Store[driver.SchedulingPolicy] + jobs *memstore.Store[jobRecord] + jobSeq *memstore.Store[int64] // registerMu serializes job-definition revision allocation so two concurrent // RegisterJobDefinition calls for the same name can't claim the same revision. registerMu sync.Mutex + // jobsMu serializes job submission (sequence allocation), cancel/terminate + // decisions and the event-emission bookkeeping on job records. + jobsMu sync.Mutex + + // events publishes "Batch Job State Change" events to the EventBridge + // default bus once wired by the provider factory. + events awsevents.Emitter + opts *config.Options } // New creates a new AWS Batch mock with the given configuration options. func New(opts *config.Options) *Mock { return &Mock{ - computeEnvs: memstore.New[driver.ComputeEnvironment](), - jobQueues: memstore.New[driver.JobQueue](), - jobDefs: memstore.New[driver.JobDefinition](), - jobDefMaxRev: memstore.New[int32](), - opts: opts, + computeEnvs: memstore.New[driver.ComputeEnvironment](), + jobQueues: memstore.New[driver.JobQueue](), + jobDefs: memstore.New[driver.JobDefinition](), + jobDefMaxRev: memstore.New[int32](), + schedPolicies: memstore.New[driver.SchedulingPolicy](), + jobs: memstore.New[jobRecord](), + jobSeq: memstore.New[int64](), + opts: opts, } } diff --git a/providers/aws/batch/compute_environment.go b/providers/aws/batch/compute_environment.go index 8136d760d..b17b8ed9b 100644 --- a/providers/aws/batch/compute_environment.go +++ b/providers/aws/batch/compute_environment.go @@ -33,6 +33,10 @@ func (m *Mock) CreateComputeEnvironment( return nil, cerrors.New(cerrors.InvalidArgument, "computeEnvironmentName is required") } + if err := validateTags(in.Tags); err != nil { + return nil, err + } + if in.Type != driver.CETypeManaged && in.Type != driver.CETypeUnmanaged { return nil, cerrors.Newf(cerrors.InvalidArgument, "type must be MANAGED or UNMANAGED, got %q", in.Type) } diff --git a/providers/aws/batch/events.go b/providers/aws/batch/events.go new file mode 100644 index 000000000..35d2ab091 --- /dev/null +++ b/providers/aws/batch/events.go @@ -0,0 +1,231 @@ +package batch + +import ( + "context" + "encoding/json" + "time" + + "github.com/stackshy/cloudemu/v2/internal/awsevents" + "github.com/stackshy/cloudemu/v2/services/batch/driver" +) + +// EventBridge identity of Batch's native job events. See "Job state change +// events" in the AWS Batch user guide. Batch publishes a "Batch Job State +// Change" event every time an existing job changes status, but none for the +// initial submission; the detail is the job as DescribeJobs reports it. +const ( + eventSource = "aws.batch" + eventJobStateChange = "Batch Job State Change" +) + +// jobEventDependency is a dependsOn entry of the event detail. +type jobEventDependency struct { + JobID string `json:"jobId,omitempty"` + Type string `json:"type,omitempty"` +} + +// jobEventArray is the arrayProperties section of the event detail. +type jobEventArray struct { + Size int32 `json:"size,omitempty"` + Index *int32 `json:"index,omitempty"` + StatusSummary map[string]int32 `json:"statusSummary,omitempty"` +} + +// jobStateChangeDetail is the detail payload of a "Batch Job State Change" +// event: the JobDetail shape DescribeJobs returns. +type jobStateChangeDetail struct { + JobArn string `json:"jobArn"` + JobName string `json:"jobName"` + JobID string `json:"jobId"` + JobQueue string `json:"jobQueue"` + Status string `json:"status"` + StatusReason string `json:"statusReason,omitempty"` + Attempts []any `json:"attempts"` + CreatedAt int64 `json:"createdAt"` + StartedAt int64 `json:"startedAt,omitempty"` + StoppedAt int64 `json:"stoppedAt,omitempty"` + RetryStrategy json.RawMessage `json:"retryStrategy,omitempty"` + DependsOn []jobEventDependency `json:"dependsOn"` + JobDefinition string `json:"jobDefinition"` + Parameters map[string]string `json:"parameters"` + Container json.RawMessage `json:"container,omitempty"` + NodeProperties json.RawMessage `json:"nodeProperties,omitempty"` + EcsProperties json.RawMessage `json:"ecsProperties,omitempty"` + EksProperties json.RawMessage `json:"eksProperties,omitempty"` + ArrayProperties *jobEventArray `json:"arrayProperties,omitempty"` + Timeout json.RawMessage `json:"timeout,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + PropagateTags bool `json:"propagateTags"` + PlatformCapabilities []string `json:"platformCapabilities"` + ShareIdentifier string `json:"shareIdentifier,omitempty"` + SchedulingPriority *int32 `json:"schedulingPriority,omitempty"` + IsCancelled bool `json:"isCancelled,omitempty"` + IsTerminated bool `json:"isTerminated,omitempty"` +} + +// SetEventPublisher wires the EventBridge default bus that job state changes +// are published to. Safe to leave unset. No events are emitted. +func (m *Mock) SetEventPublisher(p awsevents.Publisher) { + m.events.SetPublisher(p) +} + +// pendingJobEvent is one event computed under jobsMu and published after it is +// released. +type pendingJobEvent struct { + arn string + detail jobStateChangeDetail +} + +// jobPath is the ordered status path a job walks on success. Array parents go +// straight from PENDING to their terminal state; a job with dependencies passes +// through PENDING; others skip it. +func jobPath(rec *jobRecord) []string { + switch { + case rec.arraySize() > 0: + return []string{driver.JobStatusSubmitted, driver.JobStatusPending, driver.JobStatusSucceeded} + case len(rec.Job.DependsOn) > 0: + return []string{ + driver.JobStatusSubmitted, driver.JobStatusPending, driver.JobStatusRunnable, + driver.JobStatusStarting, driver.JobStatusRunning, driver.JobStatusSucceeded, + } + default: + return []string{ + driver.JobStatusSubmitted, driver.JobStatusRunnable, driver.JobStatusStarting, + driver.JobStatusRunning, driver.JobStatusSucceeded, + } + } +} + +// transitions returns the statuses a job passed through since last (exclusive) +// up to current (inclusive). A FAILED job reports only FAILED: the emulator +// does not reconstruct how far it got before failing. +func transitions(rec *jobRecord, last, current string) []string { + if last == current { + return nil + } + + if current == driver.JobStatusFailed { + return []string{current} + } + + path := jobPath(rec) + + from, to := -1, -1 + + for i, s := range path { + if s == last { + from = i + } + + if s == current { + to = i + } + } + + if to < 0 || to <= from { + return []string{current} + } + + return path[from+1 : to+1] +} + +// publishJobEvents brings every job's published state up to date: for each job +// whose observed status moved since the last event, it publishes one "Batch +// Job State Change" per status passed through. It runs on every scheduler Tick +// and on every Batch job API call, so each transition is published once, +// shortly after the provider clock passes it. It reports whether anything was +// published. +func (m *Mock) publishJobEvents(ctx context.Context, now time.Time) bool { + m.jobsMu.Lock() + + o := m.newObserver(now) + + var pending []pendingJobEvent + + all := m.jobs.All() + for id := range all { + rec := all[id] + if isTerminal(rec.LastEmitted) { + continue + } + + j := o.render(&rec, -1) + + steps := transitions(&rec, rec.LastEmitted, j.Status) + if len(steps) == 0 { + continue + } + + for _, s := range steps { + pending = append(pending, pendingJobEvent{arn: j.JobARN, detail: eventDetail(&j, s)}) + } + + next := cloneJobRecord(rec) + next.LastEmitted = j.Status + m.jobs.Set(rec.Job.JobID, next) + } + + m.jobsMu.Unlock() + + for i := range pending { + m.events.Emit(ctx, eventSource, eventJobStateChange, pending[i].detail, pending[i].arn) + } + + return len(pending) > 0 +} + +// Tick publishes the state-change events of every job whose status moved by +// now. It has the config.Tickable signature of the shared serve scheduler, so a +// job that nobody polls still reports each transition shortly after it +// happens; Batch job API calls also publish, so nothing has to call it. +func (m *Mock) Tick(now time.Time) bool { + return m.publishJobEvents(context.Background(), now) +} + +// eventDetail renders j as it stood in status: times not yet reached in that +// status are dropped. +func eventDetail(j *driver.Job, status string) jobStateChangeDetail { + d := jobStateChangeDetail{ + JobArn: j.JobARN, JobName: j.JobName, JobID: j.JobID, JobQueue: j.JobQueue, + Status: status, Attempts: []any{}, CreatedAt: j.CreatedAt, + RetryStrategy: j.RetryStrategy, DependsOn: make([]jobEventDependency, 0, len(j.DependsOn)), + JobDefinition: j.JobDefinition, Parameters: j.Parameters, Container: j.Container, + NodeProperties: j.NodeProperties, EcsProperties: j.EcsProperties, EksProperties: j.EksProperties, + Timeout: j.Timeout, Tags: j.Tags, PlatformCapabilities: j.PlatformCapabilities, + ShareIdentifier: j.ShareIdentifier, SchedulingPriority: j.SchedulingPriority, + } + + if d.Parameters == nil { + d.Parameters = map[string]string{} + } + + if d.PlatformCapabilities == nil { + d.PlatformCapabilities = []string{} + } + + if j.PropagateTags != nil { + d.PropagateTags = *j.PropagateTags + } + + for _, dep := range j.DependsOn { + d.DependsOn = append(d.DependsOn, jobEventDependency(dep)) + } + + if ap := j.ArrayProperties; ap != nil { + d.ArrayProperties = &jobEventArray{Size: ap.Size, Index: ap.Index, StatusSummary: ap.StatusSummary} + } + + if status != j.Status { + // An intermediate status replayed after the fact: report only what was + // true then. + return d + } + + d.StatusReason = j.StatusReason + d.StartedAt = j.StartedAt + d.StoppedAt = j.StoppedAt + d.IsCancelled = j.IsCancelled + d.IsTerminated = j.IsTerminated + + return d +} diff --git a/providers/aws/batch/job_definition.go b/providers/aws/batch/job_definition.go index 032b1fe69..6c034bd38 100644 --- a/providers/aws/batch/job_definition.go +++ b/providers/aws/batch/job_definition.go @@ -38,8 +38,12 @@ func (m *Mock) RegisterJobDefinition( return nil, cerrors.New(cerrors.InvalidArgument, "jobDefinitionName is required") } - if in.Type == "" { - return nil, cerrors.New(cerrors.InvalidArgument, "type is required (container or multinode)") + if err := validateTags(in.Tags); err != nil { + return nil, err + } + + if err := validateJobDefinitionType(&in); err != nil { + return nil, err } caps := in.PlatformCapabilities @@ -180,3 +184,28 @@ func (m *Mock) sortedJobDefs() []driver.JobDefinition { return out } + +// validateJobDefinitionType enforces the RegisterJobDefinition type enum +// (container | multinode) and the properties each type requires: a container +// job definition needs containerProperties, ecsProperties or eksProperties; a +// multinode one needs nodeProperties. +func validateJobDefinitionType(in *driver.RegisterJobDefinitionInput) error { + switch in.Type { + case "": + return cerrors.New(cerrors.InvalidArgument, "type is required (container or multinode)") + case driver.JDTypeContainer: + if !isSet(in.ContainerProperties) && !isSet(in.EcsProperties) && !isSet(in.EksProperties) { + return cerrors.New(cerrors.InvalidArgument, + "a container job definition requires containerProperties, ecsProperties or eksProperties") + } + case driver.JDTypeMultiNode: + if !isSet(in.NodeProperties) { + return cerrors.New(cerrors.InvalidArgument, "a multinode job definition requires nodeProperties") + } + default: + return cerrors.Newf(cerrors.InvalidArgument, + "invalid job definition type %q; valid values are container and multinode", in.Type) + } + + return nil +} diff --git a/providers/aws/batch/job_lifecycle.go b/providers/aws/batch/job_lifecycle.go new file mode 100644 index 000000000..dd2536897 --- /dev/null +++ b/providers/aws/batch/job_lifecycle.go @@ -0,0 +1,529 @@ +package batch + +import ( + "encoding/json" + "strconv" + "strings" + "time" + + "github.com/stackshy/cloudemu/v2/services/batch/driver" +) + +// Status reasons the emulator reports. "Essential container in task exited" is +// what Batch reports once a job's container exits; "Dependent Job failed" is +// the reason a job fails because one of its dependencies failed. +const ( + reasonContainerExited = "Essential container in task exited" + reasonDependentFailed = "Dependent Job failed" +) + +// Phase offsets, in units of a job's phase duration, measured from the instant +// it became ready (dependencies met): RUNNABLE for one phase, STARTING for the +// next, RUNNING for the third, then SUCCEEDED. +const ( + phasesToStarting = 1 + phasesToRunning = 2 + phasesToSucceeded = 3 +) + +// jobStop records a CancelJob/TerminateJob that took effect on a job (or one +// array child) at At. StartedAt is when that unit had started RUNNING (zero if +// it never did), so a terminated running job keeps its startedAt. +type jobStop struct { + At time.Time `json:"at"` + Reason string `json:"reason"` + Terminate bool `json:"terminate"` + StartedAt time.Time `json:"startedAt,omitempty"` +} + +// jobRecord is a stored job. Job holds the static detail captured at +// submission; status, statusReason and the started/stopped times are derived +// on every read from SubmittedAt, Phase and the job's dependencies, the way the +// settle overlay derives other services' transient states: no goroutines or +// timers, advancing the clock is what moves a job through its lifecycle. +// +// Phase is the dwell time in each non-terminal phase; zero (AsyncSettle off) +// means the job is SUCCEEDED as soon as its dependencies are. Array children +// are not stored individually: child i's state is derived from the parent +// record, with ChildStops (keyed by the decimal index) holding any child that +// was canceled or terminated. +type jobRecord struct { + Job driver.Job `json:"job"` + Seq int64 `json:"seq"` + SubmittedAt time.Time `json:"submittedAt"` + Phase time.Duration `json:"phase"` + Sequential bool `json:"sequential,omitempty"` + Stop *jobStop `json:"stop,omitempty"` + ChildStops map[string]jobStop `json:"childStops,omitempty"` + LastEmitted string `json:"lastEmitted,omitempty"` +} + +//nolint:gocritic // clone intentionally takes a value to return an independent copy-on-write copy +func cloneJobRecord(r jobRecord) jobRecord { + r.Job = cloneJob(r.Job) + + if r.Stop != nil { + s := *r.Stop + r.Stop = &s + } + + if r.ChildStops != nil { + cs := make(map[string]jobStop, len(r.ChildStops)) + for k, v := range r.ChildStops { + cs[k] = v + } + + r.ChildStops = cs + } + + return r +} + +//nolint:gocritic // clone intentionally takes a value to return an independent copy-on-write copy +func cloneJob(j driver.Job) driver.Job { + if j.DependsOn != nil { + deps := make([]driver.JobDependency, len(j.DependsOn)) + copy(deps, j.DependsOn) + j.DependsOn = deps + } + + if j.ArrayProperties != nil { + ap := *j.ArrayProperties + ap.Index = copyInt32(ap.Index) + + if ap.StatusSummary != nil { + ss := make(map[string]int32, len(ap.StatusSummary)) + for k, v := range ap.StatusSummary { + ss[k] = v + } + + ap.StatusSummary = ss + } + + j.ArrayProperties = &ap + } + + j.Parameters = copyTags(j.Parameters) + j.Container = copyRaw(j.Container) + j.NodeProperties = copyRaw(j.NodeProperties) + j.EcsProperties = copyRaw(j.EcsProperties) + j.EksProperties = copyRaw(j.EksProperties) + j.RetryStrategy = copyRaw(j.RetryStrategy) + j.Timeout = copyRaw(j.Timeout) + j.PlatformCapabilities = copyStrings(j.PlatformCapabilities) + j.PropagateTags = copyBool(j.PropagateTags) + j.SchedulingPriority = copyInt32(j.SchedulingPriority) + j.Tags = copyTags(j.Tags) + + return j +} + +func (r *jobRecord) arraySize() int { + if r.Job.ArrayProperties == nil { + return 0 + } + + return int(r.Job.ArrayProperties.Size) +} + +// jobView is a job's (or array child's) state at one observation instant. +type jobView struct { + Status string + Reason string + StartedAt time.Time + StoppedAt time.Time + Stop *jobStop +} + +func isTerminal(status string) bool { + return status == driver.JobStatusSucceeded || status == driver.JobStatusFailed +} + +// cancellable reports whether CancelJob affects a job in status: only jobs that +// haven't reached STARTING are canceled. +func cancellable(status string) bool { + switch status { + case driver.JobStatusSubmitted, driver.JobStatusPending, driver.JobStatusRunnable: + return true + default: + return false + } +} + +func later(a, b time.Time) time.Time { + if b.After(a) { + return b + } + + return a +} + +func stoppedView(s *jobStop) jobView { + return jobView{Status: driver.JobStatusFailed, Reason: s.Reason, StartedAt: s.StartedAt, StoppedAt: s.At, Stop: s} +} + +// observer derives job states at one instant, memoizing each job (and each +// array's children) so a dependency chain or a large array is evaluated once +// per call. It reads the job store without locks of its own; callers that +// mutate records hold jobsMu. +type observer struct { + m *Mock + now time.Time + views map[string]jobView + children map[string][]jobView +} + +func (m *Mock) newObserver(now time.Time) *observer { + return &observer{m: m, now: now, views: map[string]jobView{}, children: map[string][]jobView{}} +} + +// splitChildID splits "parent:index" into its parts. +func splitChildID(id string) (parent string, idx int, ok bool) { + i := strings.LastIndex(id, ":") + if i <= 0 { + return "", 0, false + } + + n, err := strconv.Atoi(id[i+1:]) + if err != nil || n < 0 { + return "", 0, false + } + + return id[:i], n, true +} + +// lookup resolves a job ID to its record and, for an array child, its index +// (-1 for a top-level job). +func (m *Mock) lookup(id string) (rec jobRecord, idx int, ok bool) { + if r, found := m.jobs.Get(id); found { + return r, -1, true + } + + parent, i, isChild := splitChildID(id) + if !isChild { + return jobRecord{}, 0, false + } + + r, found := m.jobs.Get(parent) + if !found || i >= r.arraySize() { + return jobRecord{}, 0, false + } + + return r, i, true +} + +// viewID returns the state of the job (or array child) with the given ID. +func (o *observer) viewID(id string) (jobView, bool) { + rec, idx, ok := o.m.lookup(id) + if !ok { + return jobView{}, false + } + + if idx >= 0 { + return o.childViews(&rec)[idx], true + } + + return o.view(&rec), true +} + +// view returns the state of a top-level job or array parent. +func (o *observer) view(rec *jobRecord) jobView { + if v, ok := o.views[rec.Job.JobID]; ok { + return v + } + + var v jobView + + switch { + case rec.arraySize() > 0: + v = o.parentView(rec) + case rec.Stop != nil: + v = stoppedView(rec.Stop) + default: + v = o.natural(rec, -1, nil) + } + + o.views[rec.Job.JobID] = v + + return v +} + +// childViews returns the state of every child of an array job, in index order. +func (o *observer) childViews(rec *jobRecord) []jobView { + if cv, ok := o.children[rec.Job.JobID]; ok { + return cv + } + + n := rec.arraySize() + cv := make([]jobView, n) + + for i := 0; i < n; i++ { + if s, ok := rec.ChildStops[strconv.Itoa(i)]; ok { + cv[i] = stoppedView(&s) + + continue + } + + var prev *jobView + if rec.Sequential && i > 0 { + prev = &cv[i-1] + } + + cv[i] = o.natural(rec, i, prev) + } + + o.children[rec.Job.JobID] = cv + + return cv +} + +// childAggregate summarizes an array's children. +type childAggregate struct { + started, stopped time.Time + failed, open bool +} + +func aggregateChildren(cv []jobView) childAggregate { + var a childAggregate + + for i := range cv { + c := &cv[i] + if !isTerminal(c.Status) { + a.open = true + + continue + } + + a.failed = a.failed || c.Status == driver.JobStatusFailed + + if !c.StartedAt.IsZero() && (a.started.IsZero() || c.StartedAt.Before(a.started)) { + a.started = c.StartedAt + } + + a.stopped = later(a.stopped, c.StoppedAt) + } + + return a +} + +// parentView aggregates an array parent from its children: the parent stays +// PENDING while any child is unfinished, then SUCCEEDED if every child +// succeeded and FAILED otherwise. +func (o *observer) parentView(rec *jobRecord) jobView { + a := aggregateChildren(o.childViews(rec)) + + if a.open { + if o.now.Before(rec.SubmittedAt.Add(rec.Phase)) { + return jobView{Status: driver.JobStatusSubmitted} + } + + return jobView{Status: driver.JobStatusPending} + } + + v := jobView{Status: driver.JobStatusSucceeded, StartedAt: a.started, StoppedAt: a.stopped, Stop: rec.Stop} + if a.failed { + v.Status = driver.JobStatusFailed + } + + if rec.Stop != nil { + v.Reason = rec.Stop.Reason + } + + return v +} + +// depState folds one dependency into the readiness computation: it returns the +// later ready time, whether the job must keep waiting, and whether the +// dependency failed. +func depState(dv *jobView, ready time.Time) (next time.Time, waiting, failed bool) { + switch dv.Status { + case driver.JobStatusSucceeded: + return later(ready, dv.StoppedAt), false, false + case driver.JobStatusFailed: + return later(ready, dv.StoppedAt), false, true + default: + return ready, true, false + } +} + +// dependencyID returns the job a dependency waits on for array child idx: an +// N_TO_N dependency on an array job pairs child idx with the dependency's child +// idx; anything else waits on the dependency job as a whole. +func (o *observer) dependencyID(dep driver.JobDependency, idx int) string { + if dep.Type != driver.DependencyNToN || idx < 0 { + return dep.JobID + } + + if r, ok := o.m.jobs.Get(dep.JobID); ok && idx < r.arraySize() { + return dep.JobID + ":" + strconv.Itoa(idx) + } + + return dep.JobID +} + +// natural derives the state of a job (idx -1) or array child (idx >= 0) that +// was not canceled or terminated. prev is the previous child of a SEQUENTIAL +// array. Each non-terminal phase lasts rec.Phase: SUBMITTED from submission, +// PENDING while a dependency is unfinished, then RUNNABLE, STARTING and RUNNING +// before SUCCEEDED. +func (o *observer) natural(rec *jobRecord, idx int, prev *jobView) jobView { + ready := rec.SubmittedAt.Add(rec.Phase) + if o.now.Before(ready) { + return jobView{Status: driver.JobStatusSubmitted} + } + + ready, blocked, done := o.readiness(rec, idx, prev, ready) + if done { + return blocked + } + + return o.phaseView(ready, rec.Phase) +} + +// readiness folds the job's dependencies (and a SEQUENTIAL predecessor) into +// the instant it became ready. done reports that the dependencies decide the +// state on their own (PENDING, or FAILED because one failed), returned as view. +func (o *observer) readiness(rec *jobRecord, idx int, prev *jobView, ready time.Time) (time.Time, jobView, bool) { + deps := make([]jobView, 0, len(rec.Job.DependsOn)+1) + + for _, dep := range rec.Job.DependsOn { + if dep.JobID == "" { + continue + } + + if dv, ok := o.viewID(o.dependencyID(dep, idx)); ok { + deps = append(deps, dv) + } + } + + if prev != nil { + deps = append(deps, *prev) + } + + waiting := false + + for i := range deps { + next, wait, failed := depState(&deps[i], ready) + if failed { + return ready, jobView{Status: driver.JobStatusFailed, Reason: reasonDependentFailed, StoppedAt: next}, true + } + + ready = next + waiting = waiting || wait + } + + if waiting { + return ready, jobView{Status: driver.JobStatusPending}, true + } + + return ready, jobView{}, false +} + +// phaseView places a ready job in its RUNNABLE/STARTING/RUNNING/SUCCEEDED +// phase given the phase duration d. +func (o *observer) phaseView(ready time.Time, d time.Duration) jobView { + started := ready.Add(phasesToRunning * d) + stopped := ready.Add(phasesToSucceeded * d) + + switch { + case o.now.Before(ready.Add(phasesToStarting * d)): + return jobView{Status: driver.JobStatusRunnable} + case o.now.Before(started): + return jobView{Status: driver.JobStatusStarting} + case o.now.Before(stopped): + return jobView{Status: driver.JobStatusRunning, StartedAt: started} + default: + return jobView{ + Status: driver.JobStatusSucceeded, Reason: reasonContainerExited, + StartedAt: started, StoppedAt: stopped, + } + } +} + +func millis(t time.Time) int64 { + if t.IsZero() { + return 0 + } + + return t.UnixMilli() +} + +// statusSummary counts an array's children per status, listing every status. +func statusSummary(cv []jobView) map[string]int32 { + out := map[string]int32{ + driver.JobStatusSubmitted: 0, driver.JobStatusPending: 0, driver.JobStatusRunnable: 0, + driver.JobStatusStarting: 0, driver.JobStatusRunning: 0, driver.JobStatusSucceeded: 0, + driver.JobStatusFailed: 0, + } + + for i := range cv { + out[cv[i].Status]++ + } + + return out +} + +// withExitCode returns the container detail with exitCode 0 added, as Batch +// reports for a job whose container exited successfully. +func withExitCode(container json.RawMessage) json.RawMessage { + if !isSet(container) { + return container + } + + var obj map[string]json.RawMessage + if err := json.Unmarshal(container, &obj); err != nil { + return container + } + + obj["exitCode"] = json.RawMessage("0") + + out, err := json.Marshal(obj) + if err != nil { + return container + } + + return out +} + +// render builds the externally visible job (top-level, idx -1, or array child +// idx) as observed by o. +func (o *observer) render(rec *jobRecord, idx int) driver.Job { + j := cloneJob(rec.Job) + + var v jobView + + if idx >= 0 { + v = o.childViews(rec)[idx] + j.JobID = rec.Job.JobID + ":" + strconv.Itoa(idx) + j.JobARN = rec.Job.JobARN + ":" + strconv.Itoa(idx) + i := int32(idx) + j.ArrayProperties = &driver.ArrayProperties{Index: &i} + } else { + v = o.view(rec) + + if rec.arraySize() > 0 { + j.ArrayProperties.StatusSummary = statusSummary(o.childViews(rec)) + } + } + + j.Status = v.Status + j.StatusReason = v.Reason + j.StartedAt = millis(v.StartedAt) + j.StoppedAt = millis(v.StoppedAt) + + stop := v.Stop + if idx < 0 && rec.Stop != nil { + stop = rec.Stop + } + + if stop != nil { + j.IsCancelled = !stop.Terminate + j.IsTerminated = stop.Terminate + } + + if v.Status == driver.JobStatusSucceeded && (idx >= 0 || rec.arraySize() == 0) { + j.Container = withExitCode(j.Container) + } + + return j +} diff --git a/providers/aws/batch/job_queue.go b/providers/aws/batch/job_queue.go index dd0508cb4..9be8a1b64 100644 --- a/providers/aws/batch/job_queue.go +++ b/providers/aws/batch/job_queue.go @@ -38,6 +38,10 @@ func (m *Mock) CreateJobQueue(_ context.Context, in driver.CreateJobQueueInput) return nil, cerrors.New(cerrors.InvalidArgument, "jobQueueName is required") } + if err := validateTags(in.Tags); err != nil { + return nil, err + } + if len(in.ComputeEnvironmentOrder) == 0 { return nil, cerrors.New(cerrors.InvalidArgument, "computeEnvironmentOrder must list at least one compute environment") } @@ -46,6 +50,17 @@ func (m *Mock) CreateJobQueue(_ context.Context, in driver.CreateJobQueueInput) return nil, err } + policyARN := "" + + if in.SchedulingPolicyARN != "" { + arn, err := m.resolveSchedulingPolicy(in.SchedulingPolicyARN) + if err != nil { + return nil, err + } + + policyARN = arn + } + state := in.State if state == "" { state = driver.StateEnabled @@ -59,7 +74,7 @@ func (m *Mock) CreateJobQueue(_ context.Context, in driver.CreateJobQueueInput) StatusReason: jqHealthyReason, Priority: in.Priority, ComputeEnvironmentOrder: cloneOrder(in.ComputeEnvironmentOrder), - SchedulingPolicyARN: in.SchedulingPolicyARN, + SchedulingPolicyARN: policyARN, Tags: copyTags(in.Tags), } @@ -88,9 +103,19 @@ func (m *Mock) UpdateJobQueue(_ context.Context, in driver.UpdateJobQueueInput) } } + cur, ok := m.jobQueues.Get(name) + if !ok { + return nil, cerrors.Newf(cerrors.NotFound, "job queue %q does not exist", name) + } + + policyARN, err := m.nextSchedulingPolicy(&cur, in.SchedulingPolicyARN) + if err != nil { + return nil, err + } + var updated driver.JobQueue - ok := m.jobQueues.Update(name, func(q driver.JobQueue) driver.JobQueue { + ok = m.jobQueues.Update(name, func(q driver.JobQueue) driver.JobQueue { next := cloneJobQueue(q) if in.State != nil { next.State = *in.State @@ -100,9 +125,7 @@ func (m *Mock) UpdateJobQueue(_ context.Context, in driver.UpdateJobQueueInput) next.Priority = *in.Priority } - if in.SchedulingPolicyARN != nil { - next.SchedulingPolicyARN = *in.SchedulingPolicyARN - } + next.SchedulingPolicyARN = policyARN if len(in.ComputeEnvironmentOrder) > 0 { next.ComputeEnvironmentOrder = cloneOrder(in.ComputeEnvironmentOrder) @@ -159,3 +182,26 @@ func (m *Mock) validateOrder(order []driver.ComputeEnvironmentOrder) error { return nil } + +// nextSchedulingPolicy resolves the scheduling-policy ARN an UpdateJobQueue +// leaves on the queue. Per the CreateJobQueue reference, a FIFO queue (no +// policy) can't have one added, and a fair-share queue's policy can be replaced +// but not removed. A nil ref leaves the policy unchanged. +func (m *Mock) nextSchedulingPolicy(q *driver.JobQueue, ref *string) (string, error) { + if ref == nil { + return q.SchedulingPolicyARN, nil + } + + switch { + case *ref == "" && q.SchedulingPolicyARN == "": + return "", nil + case *ref == "": + return "", cerrors.Newf(cerrors.InvalidArgument, + "the scheduling policy of job queue %q can be replaced but not removed", q.Name) + case q.SchedulingPolicyARN == "": + return "", cerrors.Newf(cerrors.InvalidArgument, + "job queue %q has no scheduling policy (FIFO) and can't have one added", q.Name) + } + + return m.resolveSchedulingPolicy(*ref) +} diff --git a/providers/aws/batch/jobs.go b/providers/aws/batch/jobs.go new file mode 100644 index 000000000..49200f87e --- /dev/null +++ b/providers/aws/batch/jobs.go @@ -0,0 +1,941 @@ +package batch + +import ( + "context" + "encoding/base64" + "encoding/json" + "regexp" + "sort" + "strconv" + "strings" + "time" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/idgen" + "github.com/stackshy/cloudemu/v2/internal/settle" + "github.com/stackshy/cloudemu/v2/services/batch/driver" +) + +// SubmitJob / CancelJob / TerminateJob / DescribeJobs bounds from the Batch API +// reference. +const ( + minArraySize = 2 + maxArraySize = 10000 + maxDependencies = 20 + maxSchedulingPriority = 9999 + minAttemptDuration = 60 + maxReasonLen = 1024 + maxDescribeJobs = 100 + + jobSeqKey = "seq" +) + +// jobNameRE is the SubmitJob jobName rule: up to 128 characters, the first +// alphanumeric, then letters, numbers, hyphens and underscores. +var jobNameRE = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$`) + +// SubmitJob validates and records a job. The job then advances through its +// lifecycle on the configured clock (see natural). Jobs run no workload: a job +// whose dependencies succeed always succeeds. +// +//nolint:gocritic // in is the driver SubmitJobInput, taken by value to match the interface +func (m *Mock) SubmitJob(ctx context.Context, in driver.SubmitJobInput) (*driver.Job, error) { + if err := validateSubmitShape(&in); err != nil { + return nil, err + } + + q, err := m.submitQueue(in.JobQueue) + if err != nil { + return nil, err + } + + if shareErr := validateShare(&q, &in); shareErr != nil { + return nil, shareErr + } + + jd, err := m.resolveJobDefinition(in.JobDefinition) + if err != nil { + return nil, err + } + + job, err := m.buildJob(&in, &q, &jd) + if err != nil { + return nil, err + } + + m.jobsMu.Lock() + + if err := m.validateDependencies(in.DependsOn); err != nil { + m.jobsMu.Unlock() + + return nil, err + } + + seq, _ := m.jobSeq.Get(jobSeqKey) + seq++ + m.jobSeq.Set(jobSeqKey, seq) + + now := m.opts.Clock.Now() + job.CreatedAt = now.UnixMilli() + + rec := jobRecord{ + Job: job, + Seq: seq, + SubmittedAt: now, + Phase: m.opts.SettleDuration(settle.DefaultBatchJobPhase), + Sequential: hasSequential(in.DependsOn), + LastEmitted: driver.JobStatusSubmitted, + } + + m.jobs.Set(job.JobID, rec) + + m.jobsMu.Unlock() + + m.publishJobEvents(ctx, m.opts.Clock.Now()) + + out := cloneJob(job) + + return &out, nil +} + +// validateSubmitShape checks the request fields that need no stored state. +func validateSubmitShape(in *driver.SubmitJobInput) error { + if err := validateSubmitRequired(in); err != nil { + return err + } + + if err := validateSubmitLimits(in); err != nil { + return err + } + + if err := validateTimeout(in.Timeout); err != nil { + return err + } + + return validateDependencyShape(in.DependsOn, in.ArraySize != nil) +} + +// validateSubmitRequired checks the required SubmitJob fields and the job name +// rule. +func validateSubmitRequired(in *driver.SubmitJobInput) error { + switch { + case in.JobName == "": + return cerrors.New(cerrors.InvalidArgument, "jobName is required") + case !jobNameRE.MatchString(in.JobName): + return cerrors.Newf(cerrors.InvalidArgument, + "jobName %q must be up to 128 characters, start with a letter or number, "+ + "and contain only letters, numbers, hyphens and underscores", in.JobName) + case in.JobQueue == "": + return cerrors.New(cerrors.InvalidArgument, "jobQueue is required") + case in.JobDefinition == "": + return cerrors.New(cerrors.InvalidArgument, "jobDefinition is required") + default: + return nil + } +} + +// validateSubmitLimits checks the documented SubmitJob numeric bounds and the +// job's tags. +func validateSubmitLimits(in *driver.SubmitJobInput) error { + if in.ArraySize != nil && (*in.ArraySize < minArraySize || *in.ArraySize > maxArraySize) { + return cerrors.Newf(cerrors.InvalidArgument, + "arrayProperties.size must be between %d and %d, got %d", minArraySize, maxArraySize, *in.ArraySize) + } + + if len(in.DependsOn) > maxDependencies { + return cerrors.Newf(cerrors.InvalidArgument, + "a job can depend on at most %d jobs, got %d", maxDependencies, len(in.DependsOn)) + } + + if p := in.SchedulingPriorityOverride; p != nil && (*p < 0 || *p > maxSchedulingPriority) { + return cerrors.Newf(cerrors.InvalidArgument, + "schedulingPriorityOverride must be between 0 and %d", maxSchedulingPriority) + } + + return validateTags(in.Tags) +} + +// validateTimeout enforces the 60-second minimum attemptDurationSeconds. +func validateTimeout(raw json.RawMessage) error { + if !isSet(raw) { + return nil + } + + var t struct { + AttemptDurationSeconds *int64 `json:"attemptDurationSeconds"` + } + + if err := json.Unmarshal(raw, &t); err != nil { + return cerrors.Newf(cerrors.InvalidArgument, "invalid timeout: %v", err) + } + + if t.AttemptDurationSeconds != nil && *t.AttemptDurationSeconds < minAttemptDuration { + return cerrors.Newf(cerrors.InvalidArgument, + "timeout.attemptDurationSeconds must be at least %d seconds", minAttemptDuration) + } + + return nil +} + +// validateDependencyShape checks dependency types: a dependency without a job +// ID is only meaningful as an array job's SEQUENTIAL dependency, and the +// N_TO_N/SEQUENTIAL array dependency types apply only to array jobs. +func validateDependencyShape(deps []driver.JobDependency, isArray bool) error { + for _, d := range deps { + switch d.Type { + case "": + case driver.DependencyNToN, driver.DependencySequential: + if !isArray { + return cerrors.Newf(cerrors.InvalidArgument, "dependency type %s is only valid for array jobs", d.Type) + } + default: + return cerrors.Newf(cerrors.InvalidArgument, + "invalid dependency type %q; valid values are N_TO_N and SEQUENTIAL", d.Type) + } + + if d.JobID == "" && d.Type != driver.DependencySequential { + return cerrors.New(cerrors.InvalidArgument, "dependsOn.jobId is required unless the type is SEQUENTIAL") + } + } + + return nil +} + +func hasSequential(deps []driver.JobDependency) bool { + for _, d := range deps { + if d.Type == driver.DependencySequential { + return true + } + } + + return false +} + +// validateDependencies requires every referenced job to exist. Callers hold +// jobsMu. +func (m *Mock) validateDependencies(deps []driver.JobDependency) error { + for _, d := range deps { + if d.JobID == "" { + continue + } + + if _, _, ok := m.lookup(d.JobID); !ok { + return cerrors.Newf(cerrors.InvalidArgument, "dependency job %q does not exist", d.JobID) + } + } + + return nil +} + +// submitQueue resolves the target queue, which must exist and be ENABLED (a +// DISABLED queue accepts no new jobs). +func (m *Mock) submitQueue(ref string) (driver.JobQueue, error) { + q, ok := m.jobQueues.Get(nameFromARN(kindJobQueue, ref)) + if !ok { + return driver.JobQueue{}, cerrors.Newf(cerrors.InvalidArgument, "job queue %q does not exist", ref) + } + + if q.State != driver.StateEnabled { + return driver.JobQueue{}, cerrors.Newf(cerrors.InvalidArgument, + "job queue %q is %s and can't accept new jobs", q.Name, q.State) + } + + return q, nil +} + +// validateShare enforces the shareIdentifier rule: required on a queue with a +// fair-share scheduling policy, forbidden on a FIFO queue. +func validateShare(q *driver.JobQueue, in *driver.SubmitJobInput) error { + if q.SchedulingPolicyARN != "" && in.ShareIdentifier == "" { + return cerrors.Newf(cerrors.InvalidArgument, + "job queue %q has a scheduling policy, so shareIdentifier must be specified", q.Name) + } + + if q.SchedulingPolicyARN == "" && in.ShareIdentifier != "" { + return cerrors.Newf(cerrors.InvalidArgument, + "job queue %q has no scheduling policy, so shareIdentifier must not be specified", q.Name) + } + + return nil +} + +// resolveJobDefinition resolves name, name:revision, or an ARN with or without +// a revision. Without a revision the latest ACTIVE revision is used. +func (m *Mock) resolveJobDefinition(ref string) (driver.JobDefinition, error) { + key := nameFromARN(kindJobDefinition, ref) + + if strings.Contains(key, ":") { + jd, ok := m.jobDefs.Get(key) + if !ok { + return driver.JobDefinition{}, cerrors.Newf(cerrors.InvalidArgument, "job definition %q does not exist", ref) + } + + return jd, nil + } + + var ( + latest driver.JobDefinition + found bool + ) + + defs := m.sortedJobDefs() + for i := range defs { + if defs[i].Name == key && defs[i].Status == driver.JDStatusActive { + latest, found = defs[i], true + } + } + + if !found { + return driver.JobDefinition{}, cerrors.Newf(cerrors.InvalidArgument, + "job definition %q has no ACTIVE revision", ref) + } + + return latest, nil +} + +// buildJob assembles the static detail of a new job from the request, the +// queue and the job definition (submit-time values override the definition's). +func (m *Mock) buildJob(in *driver.SubmitJobInput, q *driver.JobQueue, jd *driver.JobDefinition) (driver.Job, error) { + container, err := mergeContainer(jd.ContainerProperties, in.ContainerOverrides) + if err != nil { + return driver.Job{}, err + } + + id := idgen.UUID() + + job := driver.Job{ + JobID: id, + JobARN: m.arn(kindJob, id), + JobName: in.JobName, + JobQueue: q.ARN, + JobDefinition: jd.ARN, + DependsOn: cloneDeps(in.DependsOn), + Parameters: mergeParams(jd.Parameters, in.Parameters), + Container: container, + NodeProperties: copyRaw(jd.NodeProperties), + EcsProperties: copyRaw(jd.EcsProperties), + EksProperties: copyRaw(jd.EksProperties), + RetryStrategy: firstSet(in.RetryStrategy, jd.RetryStrategy), + Timeout: firstSet(in.Timeout, jd.Timeout), + PlatformCapabilities: copyStrings(jd.PlatformCapabilities), + PropagateTags: copyBool(jd.PropagateTags), + SchedulingPriority: copyInt32(jd.SchedulingPriority), + ShareIdentifier: in.ShareIdentifier, + Tags: copyTags(in.Tags), + } + + if in.PropagateTags != nil { + job.PropagateTags = copyBool(in.PropagateTags) + } + + if in.SchedulingPriorityOverride != nil { + job.SchedulingPriority = copyInt32(in.SchedulingPriorityOverride) + } + + if in.ArraySize != nil { + job.ArrayProperties = &driver.ArrayProperties{Size: *in.ArraySize} + } + + return job, nil +} + +func cloneDeps(in []driver.JobDependency) []driver.JobDependency { + if in == nil { + return nil + } + + out := make([]driver.JobDependency, len(in)) + copy(out, in) + + return out +} + +func firstSet(a, b json.RawMessage) json.RawMessage { + if isSet(a) { + return copyRaw(a) + } + + return copyRaw(b) +} + +func mergeParams(def, sub map[string]string) map[string]string { + if def == nil && sub == nil { + return nil + } + + out := copyTags(def) + if out == nil { + out = make(map[string]string, len(sub)) + } + + for k, v := range sub { + out[k] = v + } + + return out +} + +// mergeContainer applies SubmitJob containerOverrides to the job definition's +// containerProperties: environment variables and resource requirements are +// merged (by name / type), every other override replaces the property. +func mergeContainer(props, overrides json.RawMessage) (json.RawMessage, error) { + if !isSet(props) { + return nil, nil + } + + if !isSet(overrides) { + return copyRaw(props), nil + } + + var base, over map[string]json.RawMessage + + if err := json.Unmarshal(props, &base); err != nil { + return copyRaw(props), nil //nolint:nilerr // a non-object definition blob is echoed unchanged + } + + if err := json.Unmarshal(overrides, &over); err != nil { + return nil, cerrors.Newf(cerrors.InvalidArgument, "invalid containerOverrides: %v", err) + } + + for k, v := range over { + if !isSet(v) { + continue + } + + switch k { + case "environment": + base[k] = mergeKeyed(base[k], v, "name") + case "resourceRequirements": + base[k] = mergeKeyed(base[k], v, "type") + default: + base[k] = v + } + } + + out, err := json.Marshal(base) + if err != nil { + return nil, cerrors.Newf(cerrors.Internal, "merge container overrides: %v", err) + } + + return out, nil +} + +// mergeKeyed merges two JSON arrays of objects, replacing entries of base whose +// key field matches an entry of over and appending the rest. +func mergeKeyed(base, over json.RawMessage, key string) json.RawMessage { + var b, o []map[string]json.RawMessage + + if err := json.Unmarshal(over, &o); err != nil { + return over + } + + if isSet(base) { + if err := json.Unmarshal(base, &b); err != nil { + return over + } + } + + for _, entry := range o { + replaced := false + + for i := range b { + if string(b[i][key]) == string(entry[key]) { + b[i] = entry + replaced = true + + break + } + } + + if !replaced { + b = append(b, entry) + } + } + + out, err := json.Marshal(b) + if err != nil { + return over + } + + return out +} + +// DescribeJobs returns the requested jobs (top-level or array-child IDs) in +// request order, each observed now. Unknown IDs are omitted. +func (m *Mock) DescribeJobs(ctx context.Context, jobIDs []string) ([]driver.Job, error) { + if len(jobIDs) == 0 { + return nil, cerrors.New(cerrors.InvalidArgument, "jobs is required") + } + + if len(jobIDs) > maxDescribeJobs { + return nil, cerrors.Newf(cerrors.InvalidArgument, "jobs accepts at most %d job IDs, got %d", maxDescribeJobs, len(jobIDs)) + } + + m.publishJobEvents(ctx, m.opts.Clock.Now()) + + o := m.newObserver(m.opts.Clock.Now()) + out := make([]driver.Job, 0, len(jobIDs)) + + for _, id := range jobIDs { + rec, idx, ok := m.lookup(id) + if !ok { + continue + } + + out = append(out, o.render(&rec, idx)) + } + + return out, nil +} + +// ListJobs page sizes: up to 1000 results per page when listing by status and +// up to 100 when a filter is used (ListJobs API reference, maxResults). +const ( + listJobsStatusPage = 1000 + listJobsFilterPage = 100 +) + +// ListJobs lists the jobs of a queue (top-level jobs and array parents) or the +// children of an array job. Queue listings are newest first, as the ListJobs +// reference describes; children are listed by index. Exactly one of jobQueue, +// arrayJobId and multiNodeJobId must be given. Without a filter only jobs in +// jobStatus (default RUNNING) are returned. +// +// The page token is a cursor on the listing order (submission sequence, or +// child index), not an offset, so jobs submitted or changing status between +// pages never make a page skip or repeat an entry. +// +//nolint:gocritic // in is the driver ListJobsInput, taken by value to match the interface +func (m *Mock) ListJobs(ctx context.Context, in driver.ListJobsInput) (*driver.ListJobsOutput, error) { + if err := validateListJobs(&in); err != nil { + return nil, err + } + + limit, err := listJobsLimit(&in) + if err != nil { + return nil, err + } + + after, err := decodeListCursor(in.NextToken) + if err != nil { + return nil, err + } + + now := m.opts.Clock.Now() + m.publishJobEvents(ctx, now) + + l := jobLister{o: m.newObserver(now), in: &in, limit: limit, after: after} + status, useStatus := listStatus(&in) + l.status, l.useStatus = status, useStatus + + switch { + case in.MultiNodeJobID != "": + // Multi-node parallel jobs are not modeled node by node. + return &driver.ListJobsOutput{Jobs: []driver.Job{}}, nil + case in.ArrayJobID != "": + return l.children(m), nil + default: + return l.queue(m), nil + } +} + +// listJobsLimit resolves the page size: maxResults (at least 1) capped at the +// default page size for the kind of listing. +func listJobsLimit(in *driver.ListJobsInput) (int, error) { + limit := listJobsStatusPage + if len(in.Filters) > 0 { + limit = listJobsFilterPage + } + + if in.MaxResults == nil { + return limit, nil + } + + if *in.MaxResults < 1 { + return 0, cerrors.Newf(cerrors.InvalidArgument, "maxResults must be at least 1, got %d", *in.MaxResults) + } + + return min(limit, int(*in.MaxResults)), nil +} + +// decodeListCursor reads a ListJobs nextToken: the base64 of the listing +// position the previous page ended at. An empty token starts from the top. +func decodeListCursor(token string) (int64, error) { + if token == "" { + return -1, nil + } + + b, err := base64.RawURLEncoding.DecodeString(token) + if err != nil { + return 0, cerrors.New(cerrors.InvalidArgument, "invalid nextToken") + } + + n, err := strconv.ParseInt(string(b), 10, 64) + if err != nil || n < 0 { + return 0, cerrors.New(cerrors.InvalidArgument, "invalid nextToken") + } + + return n, nil +} + +func encodeListCursor(n int64) string { + return base64.RawURLEncoding.EncodeToString([]byte(strconv.FormatInt(n, 10))) +} + +// jobLister builds one ListJobs page. +type jobLister struct { + o *observer + in *driver.ListJobsInput + limit int + after int64 + status string + useStatus bool +} + +// queue lists a queue's top-level jobs newest first, resuming below the cursor +// sequence number. +func (l *jobLister) queue(m *Mock) *driver.ListJobsOutput { + queue := nameFromARN(kindJobQueue, l.in.JobQueue) + + recs := m.jobs.SortedValues() + sort.SliceStable(recs, func(i, j int) bool { return recs[i].Seq > recs[j].Seq }) + + out := &driver.ListJobsOutput{Jobs: []driver.Job{}} + + var lastSeq int64 + + for i := range recs { + rec := &recs[i] + if (l.after >= 0 && rec.Seq >= l.after) || nameFromARN(kindJobQueue, rec.Job.JobQueue) != queue { + continue + } + + j := l.o.render(rec, -1) + if (l.useStatus && j.Status != l.status) || !matchesFilters(&j, l.in.Filters) { + continue + } + + if len(out.Jobs) == l.limit { + out.NextToken = encodeListCursor(lastSeq) + + break + } + + out.Jobs = append(out.Jobs, j) + lastSeq = rec.Seq + } + + return out +} + +// children lists an array job's children by index, resuming after the cursor +// index. +func (l *jobLister) children(m *Mock) *driver.ListJobsOutput { + out := &driver.ListJobsOutput{Jobs: []driver.Job{}} + + rec, ok := m.jobs.Get(l.in.ArrayJobID) + if !ok { + return out + } + + for i := int(l.after + 1); i < rec.arraySize(); i++ { + j := l.o.render(&rec, i) + if l.useStatus && j.Status != l.status { + continue + } + + if len(out.Jobs) == l.limit { + out.NextToken = encodeListCursor(int64(*out.Jobs[len(out.Jobs)-1].ArrayProperties.Index)) + + break + } + + out.Jobs = append(out.Jobs, j) + } + + return out +} + +//nolint:gochecknoglobals // immutable enum set, read-only after init +var validJobStatuses = map[string]bool{ + driver.JobStatusSubmitted: true, driver.JobStatusPending: true, driver.JobStatusRunnable: true, + driver.JobStatusStarting: true, driver.JobStatusRunning: true, driver.JobStatusSucceeded: true, + driver.JobStatusFailed: true, +} + +//nolint:gochecknoglobals // immutable enum set, read-only after init +var validJobFilters = map[string]bool{ + driver.FilterJobName: true, driver.FilterJobDefinition: true, driver.FilterBeforeCreatedAt: true, + driver.FilterAfterCreatedAt: true, driver.FilterShareIdentifier: true, +} + +func validateListJobs(in *driver.ListJobsInput) error { + set := 0 + + for _, v := range []string{in.JobQueue, in.ArrayJobID, in.MultiNodeJobID} { + if v != "" { + set++ + } + } + + if set != 1 { + return cerrors.New(cerrors.InvalidArgument, + "specify exactly one of jobQueue, arrayJobId or multiNodeJobId") + } + + if in.JobStatus != "" && !validJobStatuses[in.JobStatus] { + return cerrors.Newf(cerrors.InvalidArgument, "invalid jobStatus %q", in.JobStatus) + } + + return validateJobFilters(in.Filters) +} + +// validateJobFilters checks filter names and values. Only one filter may be +// used at a time, except that SHARE_IDENTIFIER combines with the others. +func validateJobFilters(filters []driver.JobFilter) error { + primary := 0 + + for i := range filters { + f := &filters[i] + if !validJobFilters[f.Name] { + return cerrors.Newf(cerrors.InvalidArgument, "invalid filter name %q", f.Name) + } + + if f.Name != driver.FilterShareIdentifier { + primary++ + } + + if err := validateCreatedAtValues(f); err != nil { + return err + } + } + + if primary > 1 { + return cerrors.New(cerrors.InvalidArgument, "only one filter can be used at a time") + } + + return nil +} + +// validateCreatedAtValues requires BEFORE/AFTER_CREATED_AT values to be epoch +// milliseconds. +func validateCreatedAtValues(f *driver.JobFilter) error { + if f.Name != driver.FilterBeforeCreatedAt && f.Name != driver.FilterAfterCreatedAt { + return nil + } + + for _, v := range f.Values { + if _, err := strconv.ParseInt(v, 10, 64); err != nil { + return cerrors.Newf(cerrors.InvalidArgument, "%s value %q must be epoch milliseconds", f.Name, v) + } + } + + return nil +} + +// listStatus returns the status ListJobs filters on. A filter other than +// SHARE_IDENTIFIER disables status filtering. +func listStatus(in *driver.ListJobsInput) (string, bool) { + for _, f := range in.Filters { + if f.Name != driver.FilterShareIdentifier { + return "", false + } + } + + if in.JobStatus == "" { + return driver.JobStatusRunning, true + } + + return in.JobStatus, true +} + +func matchesFilters(j *driver.Job, filters []driver.JobFilter) bool { + for _, f := range filters { + if !matchesFilter(j, f) { + return false + } + } + + return true +} + +// matchesFilter reports whether a job matches any value of one filter. +func matchesFilter(j *driver.Job, f driver.JobFilter) bool { + for _, v := range f.Values { + if matchesFilterValue(j, f.Name, v) { + return true + } + } + + return false +} + +func matchesFilterValue(j *driver.Job, name, v string) bool { + switch name { + case driver.FilterJobName: + return matchPrefix(strings.ToLower(j.JobName), strings.ToLower(v)) + case driver.FilterJobDefinition: + if strings.HasPrefix(v, "arn:") { + return j.JobDefinition == v + } + + jdName := nameFromARN(kindJobDefinition, j.JobDefinition) + if i := strings.LastIndex(jdName, ":"); i >= 0 { + jdName = jdName[:i] + } + + return matchPrefix(jdName, v) + case driver.FilterBeforeCreatedAt: + n, _ := strconv.ParseInt(v, 10, 64) + + return j.CreatedAt < n + case driver.FilterAfterCreatedAt: + n, _ := strconv.ParseInt(v, 10, 64) + + return j.CreatedAt > n + case driver.FilterShareIdentifier: + return j.ShareIdentifier == v + default: + return false + } +} + +// matchPrefix matches s against pattern, where a trailing '*' makes the +// pattern a prefix. +func matchPrefix(s, pattern string) bool { + if p, ok := strings.CutSuffix(pattern, "*"); ok { + return strings.HasPrefix(s, p) + } + + return s == pattern +} + +// CancelJob cancels a job that hasn't reached STARTING (it moves to FAILED with +// the given reason). A job that is STARTING, RUNNING or finished is left alone +// and the call still succeeds. Canceling an array parent cancels every child +// that hasn't started. +func (m *Mock) CancelJob(ctx context.Context, jobID, reason string) error { + return m.stopJob(ctx, jobID, reason, false) +} + +// TerminateJob moves any unfinished job to FAILED with the given reason (a job +// that hadn't reached STARTING is canceled). Terminating an array parent +// terminates every unfinished child. +func (m *Mock) TerminateJob(ctx context.Context, jobID, reason string) error { + return m.stopJob(ctx, jobID, reason, true) +} + +func (m *Mock) stopJob(ctx context.Context, jobID, reason string, terminate bool) error { + if err := validateStopArgs(jobID, reason); err != nil { + return err + } + + m.jobsMu.Lock() + + rec, idx, ok := m.lookup(jobID) + if !ok { + m.jobsMu.Unlock() + + return cerrors.Newf(cerrors.NotFound, "job %q does not exist", jobID) + } + + sr := stopRequest{now: m.opts.Clock.Now(), reason: reason, terminate: terminate} + o := m.newObserver(sr.now) + + var ( + next jobRecord + changed bool + ) + + if idx >= 0 || rec.arraySize() > 0 { + next, changed = sr.applyToArray(o, &rec, idx) + } else { + next, changed = sr.applyToJob(o, &rec) + } + + if changed { + m.jobs.Set(rec.Job.JobID, next) + } + + m.jobsMu.Unlock() + + m.publishJobEvents(ctx, sr.now) + + return nil +} + +func validateStopArgs(jobID, reason string) error { + switch { + case jobID == "": + return cerrors.New(cerrors.InvalidArgument, "jobId is required") + case reason == "": + return cerrors.New(cerrors.InvalidArgument, "reason is required") + case len(reason) > maxReasonLen: + return cerrors.Newf(cerrors.InvalidArgument, "reason must be at most %d characters", maxReasonLen) + default: + return nil + } +} + +// stopRequest is one CancelJob (terminate false) or TerminateJob call. +type stopRequest struct { + now time.Time + reason string + terminate bool +} + +// affects reports whether the call stops a unit in status: TerminateJob stops +// anything unfinished, CancelJob only what hasn't reached STARTING. +func (sr *stopRequest) affects(status string) bool { + if sr.terminate { + return !isTerminal(status) + } + + return cancellable(status) +} + +// stopFor records the stop of a unit currently in v. A terminate of a unit +// that hadn't reached STARTING counts as a cancel. +func (sr *stopRequest) stopFor(v *jobView) jobStop { + return jobStop{At: sr.now, Reason: sr.reason, Terminate: sr.terminate && !cancellable(v.Status), StartedAt: v.StartedAt} +} + +func (sr *stopRequest) applyToJob(o *observer, rec *jobRecord) (jobRecord, bool) { + v := o.view(rec) + if !sr.affects(v.Status) { + return jobRecord{}, false + } + + next := cloneJobRecord(*rec) + s := sr.stopFor(&v) + next.Stop = &s + + return next, true +} + +// applyToArray stops one child (idx >= 0) or, for the parent (idx -1), every +// affected child, marking the parent canceled/terminated when any child was. +func (sr *stopRequest) applyToArray(o *observer, rec *jobRecord, idx int) (jobRecord, bool) { + next := cloneJobRecord(*rec) + cv := o.childViews(rec) + changed := false + + for i := range cv { + if (idx >= 0 && i != idx) || !sr.affects(cv[i].Status) { + continue + } + + if next.ChildStops == nil { + next.ChildStops = map[string]jobStop{} + } + + next.ChildStops[strconv.Itoa(i)] = sr.stopFor(&cv[i]) + changed = true + } + + if changed && idx < 0 { + next.Stop = &jobStop{At: sr.now, Reason: sr.reason, Terminate: sr.terminate} + } + + return next, changed +} diff --git a/providers/aws/batch/jobs_paging_test.go b/providers/aws/batch/jobs_paging_test.go new file mode 100644 index 000000000..7abb6c352 --- /dev/null +++ b/providers/aws/batch/jobs_paging_test.go @@ -0,0 +1,238 @@ +package batch_test + +import ( + "context" + "strings" + "sync" + "testing" + "time" + + "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/providers/aws/batch" + "github.com/stackshy/cloudemu/v2/services/batch/driver" +) + +func listJobs(m *batch.Mock, in driver.ListJobsInput) ([]driver.Job, error) { + out, err := m.ListJobs(context.Background(), in) + if err != nil { + return nil, err + } + + return out.Jobs, nil +} + +func TestListJobsCursorIsStable(t *testing.T) { + m := newMock(t) + setupQueue(t, m) + ctx := context.Background() + + for _, n := range []string{"a", "b", "c", "d"} { + submit(t, m, driver.SubmitJobInput{JobName: n}) + } + + two := int32(2) + in := driver.ListJobsInput{JobQueue: "q", JobStatus: driver.JobStatusSucceeded, MaxResults: &two} + + p1, err := m.ListJobs(ctx, in) + if err != nil || len(p1.Jobs) != 2 || p1.NextToken == "" || p1.Jobs[0].JobName != "d" || p1.Jobs[1].JobName != "c" { + t.Fatalf("page 1 = %+v, %v", p1, err) + } + + // A job submitted between pages must not shift the next page. + submit(t, m, driver.SubmitJobInput{JobName: "e"}) + + in.NextToken = p1.NextToken + + p2, err := m.ListJobs(ctx, in) + if err != nil || len(p2.Jobs) != 2 || p2.NextToken != "" || p2.Jobs[0].JobName != "b" || p2.Jobs[1].JobName != "a" { + t.Fatalf("page 2 = %+v, %v", p2, err) + } + + in.NextToken = "!!" + if _, err := m.ListJobs(ctx, in); !errors.IsInvalidArgument(err) { + t.Fatalf("bad token: want InvalidArgument, got %v", err) + } + + zero := int32(0) + if _, err := m.ListJobs(ctx, driver.ListJobsInput{JobQueue: "q", MaxResults: &zero}); !errors.IsInvalidArgument(err) { + t.Fatalf("maxResults 0: want InvalidArgument, got %v", err) + } + + // Array children page by index. + size := int32(5) + arr := submit(t, m, driver.SubmitJobInput{JobName: "arr", ArraySize: &size}) + cin := driver.ListJobsInput{ArrayJobID: arr.JobID, JobStatus: driver.JobStatusSucceeded, MaxResults: &two} + + var idx []int32 + + for { + page, err := m.ListJobs(ctx, cin) + if err != nil { + t.Fatal(err) + } + + for i := range page.Jobs { + idx = append(idx, *page.Jobs[i].ArrayProperties.Index) + } + + if page.NextToken == "" { + break + } + + cin.NextToken = page.NextToken + } + + if len(idx) != 5 || idx[0] != 0 || idx[4] != 4 { + t.Fatalf("children pages = %v, want 0..4", idx) + } +} + +func TestTickPublishesUnpolledTransitions(t *testing.T) { + m, fc := newAsyncMock(t) + pub := &recordingPublisher{} + m.SetEventPublisher(pub) + setupQueue(t, m) + + submit(t, m, driver.SubmitJobInput{}) + + if m.Tick(fc.Now()) { + t.Fatal("Tick right after submit reported a change") + } + + fc.Advance(2 * time.Second) + + if !m.Tick(fc.Now()) { + t.Fatal("Tick after 2s reported no change") + } + + if got := pub.statuses(); len(got) != 2 || got[0] != "RUNNABLE" || got[1] != "STARTING" { + t.Fatalf("events after Tick = %v, want [RUNNABLE STARTING]", got) + } + + fc.Advance(10 * time.Second) + m.Tick(fc.Now()) + + if got := pub.statuses(); len(got) != 4 || got[3] != "SUCCEEDED" { + t.Fatalf("events = %v, want [... RUNNING SUCCEEDED]", got) + } + + if m.Tick(fc.Now()) { + t.Fatal("Tick on a finished job reported a change") + } +} + +func TestTagValidation(t *testing.T) { + m := newMock(t) + setupQueue(t, m) + ctx := context.Background() + + j := submit(t, m, driver.SubmitJobInput{}) + + bad := []map[string]string{ + {"aws:owner": "x"}, + {"AWS:owner": "x"}, + {"": "x"}, + {strings.Repeat("k", 129): "x"}, + {"k": strings.Repeat("v", 257)}, + } + + for i, tags := range bad { + if err := m.TagResource(ctx, j.JobARN, tags); !errors.IsInvalidArgument(err) { + t.Errorf("case %d: TagResource want InvalidArgument, got %v", i, err) + } + + if _, err := m.SubmitJob(ctx, driver.SubmitJobInput{ + JobName: "t", JobQueue: "q", JobDefinition: "jd", Tags: tags, + }); !errors.IsInvalidArgument(err) { + t.Errorf("case %d: SubmitJob want InvalidArgument, got %v", i, err) + } + + if _, err := m.CreateSchedulingPolicy(ctx, driver.CreateSchedulingPolicyInput{Name: "sp", Tags: tags}); !errors.IsInvalidArgument(err) { + t.Errorf("case %d: CreateSchedulingPolicy want InvalidArgument, got %v", i, err) + } + } + + if err := m.TagResource(ctx, j.JobARN, map[string]string{"team": "a"}); err != nil { + t.Fatalf("TagResource(job): %v", err) + } + + if tags, err := m.ListTagsForResource(ctx, j.JobARN); err != nil || tags["team"] != "a" { + t.Fatalf("ListTagsForResource(job) = %v, %v", tags, err) + } + + // Child jobs of an array job can't be tagged. + size := int32(2) + arr := submit(t, m, driver.SubmitJobInput{JobName: "arr", ArraySize: &size}) + + if err := m.TagResource(ctx, arr.JobARN+":0", map[string]string{"k": "v"}); !errors.IsNotFound(err) { + t.Fatalf("tagging an array child: want NotFound, got %v", err) + } + + // The merged tag set is capped at 50. + many := map[string]string{} + for i := 0; i < 50; i++ { + many["k"+strings.Repeat("x", i)] = "v" + } + + if err := m.TagResource(ctx, j.JobARN, many); !errors.IsInvalidArgument(err) { + t.Fatalf("51 total tags: want InvalidArgument, got %v", err) + } + + if tags, _ := m.ListTagsForResource(ctx, j.JobARN); len(tags) != 1 { + t.Fatalf("rejected TagResource changed tags: %v", tags) + } +} + +// TestConcurrentJobOperations runs submit, describe, list, cancel, terminate, +// tag and Tick at once (run with -race), with an event target that calls back +// into the mock, which must not deadlock. +func TestConcurrentJobOperations(t *testing.T) { + m, fc := newAsyncMock(t) + setupQueue(t, m) + m.SetEventPublisher(callbackPublisher{m: m}) + + ctx := context.Background() + + var wg sync.WaitGroup + + for w := 0; w < 6; w++ { + wg.Add(1) + + go func() { + defer wg.Done() + + for i := 0; i < 20; i++ { + j, err := m.SubmitJob(ctx, driver.SubmitJobInput{JobName: "c", JobQueue: "q", JobDefinition: "jd"}) + if err != nil { + t.Error(err) + + return + } + + _, _ = m.DescribeJobs(ctx, []string{j.JobID}) + _, _ = m.ListJobs(ctx, driver.ListJobsInput{JobQueue: "q"}) + _ = m.TagResource(ctx, j.JobARN, map[string]string{"k": "v"}) + + if i%2 == 0 { + _ = m.CancelJob(ctx, j.JobID, "r") + } else { + _ = m.TerminateJob(ctx, j.JobID, "r") + } + + fc.Advance(100 * time.Millisecond) + m.Tick(fc.Now()) + } + }() + } + + wg.Wait() +} + +// callbackPublisher re-enters the mock from event delivery, like an +// EventBridge rule whose target calls Batch. +type callbackPublisher struct{ m *batch.Mock } + +func (p callbackPublisher) PublishServiceEvent(ctx context.Context, _, _ string, _ any, resources []string) { + _, _ = p.m.DescribeJobs(ctx, resources) + _, _ = p.m.ListJobs(ctx, driver.ListJobsInput{JobQueue: "q", JobStatus: driver.JobStatusFailed}) +} diff --git a/providers/aws/batch/jobs_test.go b/providers/aws/batch/jobs_test.go new file mode 100644 index 000000000..f22d53511 --- /dev/null +++ b/providers/aws/batch/jobs_test.go @@ -0,0 +1,726 @@ +package batch_test + +import ( + "context" + "encoding/json" + "sync" + "testing" + "time" + + "github.com/stackshy/cloudemu/v2/config" + "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/providers/aws/batch" + "github.com/stackshy/cloudemu/v2/services/batch/driver" +) + +var jobEpoch = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + +// recordingPublisher captures published EventBridge events. +type recordingPublisher struct { + mu sync.Mutex + events []recordedEvent +} + +type recordedEvent struct { + source, detailType string + detail map[string]any + resources []string +} + +func (p *recordingPublisher) PublishServiceEvent(_ context.Context, source, detailType string, detail any, resources []string) { + b, _ := json.Marshal(detail) + + var d map[string]any + _ = json.Unmarshal(b, &d) + + p.mu.Lock() + defer p.mu.Unlock() + + p.events = append(p.events, recordedEvent{source: source, detailType: detailType, detail: d, resources: resources}) +} + +func (p *recordingPublisher) statuses() []string { + p.mu.Lock() + defer p.mu.Unlock() + + out := make([]string, 0, len(p.events)) + for _, e := range p.events { + out = append(out, e.detail["status"].(string)) + } + + return out +} + +// newAsyncMock returns a mock whose jobs progress on a fake clock. +func newAsyncMock(t *testing.T) (*batch.Mock, *config.FakeClock) { + t.Helper() + + fc := config.NewFakeClock(jobEpoch) + + return batch.New(config.NewOptions( + config.WithRegion("us-east-1"), + config.WithAccountID("000000000000"), + config.WithClock(fc), + config.WithAsyncSettle(), + )), fc +} + +// setupQueue creates a compute environment, a job queue "q" and a container +// job definition "jd". +func setupQueue(t *testing.T, m *batch.Mock) { + t.Helper() + + ctx := context.Background() + mustCreateCE(t, m, "ce") + + if _, err := m.CreateJobQueue(ctx, driver.CreateJobQueueInput{ + Name: "q", + Priority: 1, + ComputeEnvironmentOrder: []driver.ComputeEnvironmentOrder{{Order: 1, ComputeEnvironment: "ce"}}, + }); err != nil { + t.Fatalf("create queue: %v", err) + } + + if _, err := m.RegisterJobDefinition(ctx, driver.RegisterJobDefinitionInput{ + Name: "jd", Type: driver.JDTypeContainer, + ContainerProperties: json.RawMessage(`{"image":"busybox","command":["echo","hi"]}`), + }); err != nil { + t.Fatalf("register: %v", err) + } +} + +func submit(t *testing.T, m *batch.Mock, in driver.SubmitJobInput) *driver.Job { + t.Helper() + + if in.JobName == "" { + in.JobName = "job" + } + + if in.JobQueue == "" { + in.JobQueue = "q" + } + + if in.JobDefinition == "" { + in.JobDefinition = "jd" + } + + j, err := m.SubmitJob(context.Background(), in) + if err != nil { + t.Fatalf("SubmitJob: %v", err) + } + + return j +} + +func describe(t *testing.T, m *batch.Mock, id string) driver.Job { + t.Helper() + + jobs, err := m.DescribeJobs(context.Background(), []string{id}) + if err != nil || len(jobs) != 1 { + t.Fatalf("DescribeJobs(%s) = %v, %v", id, jobs, err) + } + + return jobs[0] +} + +func TestSubmitJobSyncSucceedsImmediately(t *testing.T) { + m := newMock(t) + setupQueue(t, m) + + j := submit(t, m, driver.SubmitJobInput{ + ContainerOverrides: json.RawMessage(`{"command":["echo","bye"]}`), + Parameters: map[string]string{"p": "v"}, + }) + if j.JobID == "" || j.JobARN != "arn:aws:batch:us-east-1:000000000000:job/"+j.JobID { + t.Fatalf("bad identity: %+v", j) + } + + got := describe(t, m, j.JobID) + if got.Status != driver.JobStatusSucceeded { + t.Fatalf("status = %s, want SUCCEEDED", got.Status) + } + + if got.CreatedAt == 0 || got.StartedAt == 0 || got.StoppedAt == 0 { + t.Fatalf("timestamps not set: %+v", got) + } + + if got.JobQueue != "arn:aws:batch:us-east-1:000000000000:job-queue/q" || + got.JobDefinition != "arn:aws:batch:us-east-1:000000000000:job-definition/jd:1" { + t.Fatalf("queue/definition not resolved to ARNs: %s %s", got.JobQueue, got.JobDefinition) + } + + var c map[string]any + if err := json.Unmarshal(got.Container, &c); err != nil { + t.Fatalf("container: %v", err) + } + + if c["image"] != "busybox" || c["command"].([]any)[1] != "bye" || c["exitCode"] != float64(0) { + t.Fatalf("container overrides/exitCode not applied: %v", c) + } +} + +func TestJobLifecycleAdvancesWithClock(t *testing.T) { + m, fc := newAsyncMock(t) + setupQueue(t, m) + + j := submit(t, m, driver.SubmitJobInput{}) + + want := []string{ + driver.JobStatusSubmitted, driver.JobStatusRunnable, driver.JobStatusStarting, + driver.JobStatusRunning, driver.JobStatusSucceeded, + } + + for i, w := range want { + got := describe(t, m, j.JobID) + if got.Status != w { + t.Fatalf("step %d: status = %s, want %s", i, got.Status, w) + } + + if w == driver.JobStatusRunning && got.StartedAt == 0 { + t.Fatal("RUNNING job has no startedAt") + } + + if w == driver.JobStatusSucceeded && (got.StoppedAt <= got.StartedAt || got.StatusReason == "") { + t.Fatalf("SUCCEEDED job times/reason wrong: %+v", got) + } + + fc.Advance(time.Second) + } +} + +func TestSubmitJobValidation(t *testing.T) { + m := newMock(t) + setupQueue(t, m) + ctx := context.Background() + + one, tooMany := int32(1), int32(10001) + + cases := map[string]driver.SubmitJobInput{ + "missing name": {JobQueue: "q", JobDefinition: "jd"}, + "bad name": {JobName: "-bad", JobQueue: "q", JobDefinition: "jd"}, + "missing queue": {JobName: "j", JobQueue: "nope", JobDefinition: "jd"}, + "missing definition": {JobName: "j", JobQueue: "q", JobDefinition: "nope"}, + "missing revision": {JobName: "j", JobQueue: "q", JobDefinition: "jd:9"}, + "array size 1": {JobName: "j", JobQueue: "q", JobDefinition: "jd", ArraySize: &one}, + "array size 10001": {JobName: "j", JobQueue: "q", JobDefinition: "jd", ArraySize: &tooMany}, + "unknown dependency": {JobName: "j", JobQueue: "q", JobDefinition: "jd", + DependsOn: []driver.JobDependency{{JobID: "missing"}}}, + "bad dependency type": {JobName: "j", JobQueue: "q", JobDefinition: "jd", + DependsOn: []driver.JobDependency{{JobID: "x", Type: "BOGUS"}}}, + "N_TO_N on non-array": {JobName: "j", JobQueue: "q", JobDefinition: "jd", + DependsOn: []driver.JobDependency{{JobID: "x", Type: driver.DependencyNToN}}}, + "share id on FIFO queue": {JobName: "j", JobQueue: "q", JobDefinition: "jd", ShareIdentifier: "A"}, + "short timeout": {JobName: "j", JobQueue: "q", JobDefinition: "jd", + Timeout: json.RawMessage(`{"attemptDurationSeconds":30}`)}, + } + + for name, in := range cases { + if _, err := m.SubmitJob(ctx, in); !errors.IsInvalidArgument(err) { + t.Errorf("%s: want InvalidArgument, got %v", name, err) + } + } + + // A DISABLED queue accepts no new jobs. + disabled := driver.StateDisabled + if _, err := m.UpdateJobQueue(ctx, driver.UpdateJobQueueInput{Name: "q", State: &disabled}); err != nil { + t.Fatalf("disable: %v", err) + } + + if _, err := m.SubmitJob(ctx, driver.SubmitJobInput{JobName: "j", JobQueue: "q", JobDefinition: "jd"}); !errors.IsInvalidArgument(err) { + t.Fatalf("submit to DISABLED queue: want InvalidArgument, got %v", err) + } +} + +func TestSubmitJobUsesLatestActiveRevision(t *testing.T) { + m := newMock(t) + setupQueue(t, m) + ctx := context.Background() + + if _, err := m.RegisterJobDefinition(ctx, driver.RegisterJobDefinitionInput{ + Name: "jd", Type: driver.JDTypeContainer, ContainerProperties: json.RawMessage(`{"image":"v2"}`), + }); err != nil { + t.Fatal(err) + } + + if err := m.DeregisterJobDefinition(ctx, "jd:2"); err != nil { + t.Fatal(err) + } + + j := describe(t, m, submit(t, m, driver.SubmitJobInput{}).JobID) + if j.JobDefinition != "arn:aws:batch:us-east-1:000000000000:job-definition/jd:1" { + t.Fatalf("want latest ACTIVE revision 1, got %s", j.JobDefinition) + } +} + +func TestDependentJobWaitsAndFailsWithDependency(t *testing.T) { + m, fc := newAsyncMock(t) + setupQueue(t, m) + ctx := context.Background() + + parent := submit(t, m, driver.SubmitJobInput{JobName: "parent"}) + child := submit(t, m, driver.SubmitJobInput{ + JobName: "child", DependsOn: []driver.JobDependency{{JobID: parent.JobID}}, + }) + + fc.Advance(time.Second) + + if got := describe(t, m, child.JobID).Status; got != driver.JobStatusPending { + t.Fatalf("child before parent finishes = %s, want PENDING", got) + } + + // Parent finishes at +4s; child then runs RUNNABLE/STARTING/RUNNING and + // succeeds at +7s. + fc.Advance(3 * time.Second) + + if got := describe(t, m, child.JobID).Status; got != driver.JobStatusRunnable { + t.Fatalf("child after parent succeeded = %s, want RUNNABLE", got) + } + + fc.Advance(3 * time.Second) + + if got := describe(t, m, child.JobID).Status; got != driver.JobStatusSucceeded { + t.Fatalf("child = %s, want SUCCEEDED", got) + } + + // A dependency that fails fails its dependents. + p2 := submit(t, m, driver.SubmitJobInput{JobName: "p2"}) + c2 := submit(t, m, driver.SubmitJobInput{JobName: "c2", DependsOn: []driver.JobDependency{{JobID: p2.JobID}}}) + + if err := m.CancelJob(ctx, p2.JobID, "stop it"); err != nil { + t.Fatal(err) + } + + fc.Advance(time.Second) + + got := describe(t, m, c2.JobID) + if got.Status != driver.JobStatusFailed || got.StatusReason != "Dependent Job failed" { + t.Fatalf("dependent of cancelled job = %s (%q), want FAILED (Dependent Job failed)", got.Status, got.StatusReason) + } +} + +func TestCancelVersusTerminate(t *testing.T) { + m, fc := newAsyncMock(t) + setupQueue(t, m) + ctx := context.Background() + + // Cancel affects a job that hasn't reached STARTING. + runnable := submit(t, m, driver.SubmitJobInput{JobName: "runnable"}) + fc.Advance(time.Second) + + if err := m.CancelJob(ctx, runnable.JobID, "user cancel"); err != nil { + t.Fatal(err) + } + + got := describe(t, m, runnable.JobID) + if got.Status != driver.JobStatusFailed || !got.IsCancelled || got.IsTerminated || got.StatusReason != "user cancel" { + t.Fatalf("cancelled RUNNABLE job = %+v", got) + } + + // Cancel leaves a RUNNING job alone (and still succeeds). + running := submit(t, m, driver.SubmitJobInput{JobName: "running"}) + fc.Advance(3 * time.Second) + + if err := m.CancelJob(ctx, running.JobID, "too late"); err != nil { + t.Fatalf("cancel of RUNNING job should succeed, got %v", err) + } + + if got := describe(t, m, running.JobID); got.Status != driver.JobStatusRunning || got.IsCancelled { + t.Fatalf("cancel must not affect a RUNNING job: %+v", got) + } + + // Terminate stops a RUNNING job, keeping its startedAt. + if err := m.TerminateJob(ctx, running.JobID, "kill"); err != nil { + t.Fatal(err) + } + + got = describe(t, m, running.JobID) + if got.Status != driver.JobStatusFailed || !got.IsTerminated || got.StartedAt == 0 || got.StatusReason != "kill" { + t.Fatalf("terminated RUNNING job = %+v", got) + } + + // Terminate on a SUBMITTED job cancels it. + fresh := submit(t, m, driver.SubmitJobInput{JobName: "fresh"}) + if err := m.TerminateJob(ctx, fresh.JobID, "early"); err != nil { + t.Fatal(err) + } + + if got := describe(t, m, fresh.JobID); got.Status != driver.JobStatusFailed || !got.IsCancelled || got.IsTerminated { + t.Fatalf("terminate before STARTING should cancel: %+v", got) + } + + // Finished jobs are untouched; unknown jobs and missing reasons are errors. + fc.Advance(10 * time.Second) + + if err := m.TerminateJob(ctx, running.JobID, "again"); err != nil { + t.Fatal(err) + } + + if got := describe(t, m, running.JobID); got.StatusReason != "kill" { + t.Fatalf("terminating a finished job changed it: %+v", got) + } + + if err := m.CancelJob(ctx, "missing", "r"); !errors.IsNotFound(err) { + t.Fatalf("unknown job: want NotFound, got %v", err) + } + + if err := m.CancelJob(ctx, fresh.JobID, ""); !errors.IsInvalidArgument(err) { + t.Fatalf("missing reason: want InvalidArgument, got %v", err) + } +} + +func TestArrayJobChildren(t *testing.T) { + m, fc := newAsyncMock(t) + setupQueue(t, m) + ctx := context.Background() + + size := int32(3) + parent := submit(t, m, driver.SubmitJobInput{ + ArraySize: &size, DependsOn: []driver.JobDependency{{Type: driver.DependencySequential}}, + }) + + fc.Advance(2 * time.Second) + + p := describe(t, m, parent.JobID) + if p.Status != driver.JobStatusPending || p.ArrayProperties == nil || p.ArrayProperties.Size != 3 { + t.Fatalf("array parent while children run = %+v", p) + } + + if p.ArrayProperties.StatusSummary[driver.JobStatusStarting] != 1 || + p.ArrayProperties.StatusSummary[driver.JobStatusPending] != 2 { + t.Fatalf("SEQUENTIAL children should run one at a time: %v", p.ArrayProperties.StatusSummary) + } + + child := describe(t, m, parent.JobID+":0") + if child.ArrayProperties == nil || child.ArrayProperties.Index == nil || *child.ArrayProperties.Index != 0 { + t.Fatalf("child index missing: %+v", child) + } + + all := driver.JobStatusPending + kids, err := listJobs(m, driver.ListJobsInput{ArrayJobID: parent.JobID, JobStatus: all}) + if err != nil || len(kids) != 2 { + t.Fatalf("ListJobs(arrayJobId, PENDING) = %d, %v; want 2", len(kids), err) + } + + fc.Advance(20 * time.Second) + + if got := describe(t, m, parent.JobID); got.Status != driver.JobStatusSucceeded || + got.ArrayProperties.StatusSummary[driver.JobStatusSucceeded] != 3 { + t.Fatalf("array parent after children finish = %+v", got) + } + + // Terminating an array parent fails every unfinished child. + p2 := submit(t, m, driver.SubmitJobInput{JobName: "arr2", ArraySize: &size}) + if err := m.TerminateJob(ctx, p2.JobID, "abort"); err != nil { + t.Fatal(err) + } + + got := describe(t, m, p2.JobID) + if got.Status != driver.JobStatusFailed || got.ArrayProperties.StatusSummary[driver.JobStatusFailed] != 3 { + t.Fatalf("terminated array = %+v", got) + } +} + +func TestListJobs(t *testing.T) { + m, fc := newAsyncMock(t) + setupQueue(t, m) + ctx := context.Background() + + a := submit(t, m, driver.SubmitJobInput{JobName: "alpha"}) + fc.Advance(time.Second) + b := submit(t, m, driver.SubmitJobInput{JobName: "beta"}) + fc.Advance(2 * time.Second) // alpha RUNNING, beta STARTING + + running, err := listJobs(m, driver.ListJobsInput{JobQueue: "q"}) + if err != nil || len(running) != 1 || running[0].JobID != a.JobID { + t.Fatalf("default ListJobs (RUNNING) = %+v, %v", running, err) + } + + byName, err := listJobs(m, driver.ListJobsInput{ + JobQueue: "arn:aws:batch:us-east-1:000000000000:job-queue/q", + Filters: []driver.JobFilter{{Name: driver.FilterJobName, Values: []string{"BET*"}}}, + }) + if err != nil || len(byName) != 1 || byName[0].JobID != b.JobID { + t.Fatalf("JOB_NAME filter = %+v, %v", byName, err) + } + + all, err := listJobs(m, driver.ListJobsInput{ + JobQueue: "q", Filters: []driver.JobFilter{{Name: driver.FilterJobDefinition, Values: []string{"jd"}}}, + }) + if err != nil || len(all) != 2 || all[0].JobID != b.JobID { + t.Fatalf("JOB_DEFINITION filter should list both, newest first: %+v, %v", all, err) + } + + bad := []driver.ListJobsInput{ + {}, + {JobQueue: "q", ArrayJobID: "x"}, + {JobQueue: "q", JobStatus: "DONE"}, + {JobQueue: "q", Filters: []driver.JobFilter{{Name: "NOPE"}}}, + {JobQueue: "q", Filters: []driver.JobFilter{{Name: driver.FilterJobName}, {Name: driver.FilterJobDefinition}}}, + } + + for i, in := range bad { + if _, err := m.ListJobs(ctx, in); !errors.IsInvalidArgument(err) { + t.Errorf("case %d: want InvalidArgument, got %v", i, err) + } + } +} + +func TestJobStateChangeEvents(t *testing.T) { + m := newMock(t) + pub := &recordingPublisher{} + m.SetEventPublisher(pub) + setupQueue(t, m) + + j := submit(t, m, driver.SubmitJobInput{}) + + got := pub.statuses() + want := []string{"RUNNABLE", "STARTING", "RUNNING", "SUCCEEDED"} + + if len(got) != len(want) { + t.Fatalf("events = %v, want %v (no event for the initial submission)", got, want) + } + + for i := range want { + if got[i] != want[i] { + t.Fatalf("events = %v, want %v", got, want) + } + } + + e := pub.events[3] + if e.source != "aws.batch" || e.detailType != "Batch Job State Change" || + len(e.resources) != 1 || e.resources[0] != j.JobARN || e.detail["jobId"] != j.JobID { + t.Fatalf("bad event envelope: %+v", e) + } + + // Describing again publishes nothing new. + describe(t, m, j.JobID) + + if n := len(pub.statuses()); n != 4 { + t.Fatalf("re-observing a finished job published more events: %d", n) + } +} + +func TestJobEventsFollowAsyncLifecycle(t *testing.T) { + m, fc := newAsyncMock(t) + pub := &recordingPublisher{} + m.SetEventPublisher(pub) + setupQueue(t, m) + ctx := context.Background() + + j := submit(t, m, driver.SubmitJobInput{}) + if n := len(pub.statuses()); n != 0 { + t.Fatalf("submission published %d events, want 0", n) + } + + fc.Advance(time.Second) + describe(t, m, j.JobID) + + if got := pub.statuses(); len(got) != 1 || got[0] != "RUNNABLE" { + t.Fatalf("events after 1s = %v, want [RUNNABLE]", got) + } + + if err := m.CancelJob(ctx, j.JobID, "stop"); err != nil { + t.Fatal(err) + } + + if got := pub.statuses(); len(got) != 2 || got[1] != "FAILED" { + t.Fatalf("events after cancel = %v, want [RUNNABLE FAILED]", got) + } +} + +func TestSchedulingPolicyCRUDAndQueueBinding(t *testing.T) { + m := newMock(t) + ctx := context.Background() + mustCreateCE(t, m, "ce") + + p, err := m.CreateSchedulingPolicy(ctx, driver.CreateSchedulingPolicyInput{ + Name: "fair", + FairsharePolicy: json.RawMessage(`{"shareDecaySeconds":3600,"computeReservation":1,"shareDistribution":[{"shareIdentifier":"A*","weightFactor":0.5}]}`), + Tags: map[string]string{"k": "v"}, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + if p.ARN != "arn:aws:batch:us-east-1:000000000000:scheduling-policy/fair" { + t.Fatalf("arn = %s", p.ARN) + } + + invalid := []driver.CreateSchedulingPolicyInput{ + {Name: "fair"}, + {Name: "bad name"}, + {Name: "x", FairsharePolicy: json.RawMessage(`{"computeReservation":100}`)}, + {Name: "x", FairsharePolicy: json.RawMessage(`{"shareDecaySeconds":604801}`)}, + {Name: "x", FairsharePolicy: json.RawMessage(`{"shareDistribution":[{"shareIdentifier":"A","weightFactor":1000}]}`)}, + {Name: "x", FairsharePolicy: json.RawMessage(`{}`), QuotaSharePolicy: json.RawMessage(`{}`)}, + } + + for i, in := range invalid { + if _, err := m.CreateSchedulingPolicy(ctx, in); err == nil { + t.Errorf("case %d: want error", i) + } + } + + if err := m.UpdateSchedulingPolicy(ctx, driver.UpdateSchedulingPolicyInput{ + ARN: p.ARN, FairsharePolicy: json.RawMessage(`{"shareDecaySeconds":60}`), + }); err != nil { + t.Fatalf("update: %v", err) + } + + got, err := m.DescribeSchedulingPolicies(ctx, []string{p.ARN, "arn:aws:batch:us-east-1:000000000000:scheduling-policy/none"}) + if err != nil || len(got) != 1 || string(got[0].FairsharePolicy) != `{"shareDecaySeconds":60}` || got[0].Tags["k"] != "v" { + t.Fatalf("describe = %+v, %v", got, err) + } + + if _, err := m.DescribeSchedulingPolicies(ctx, nil); !errors.IsInvalidArgument(err) { + t.Fatalf("describe without arns: want InvalidArgument, got %v", err) + } + + // A queue must reference an existing policy. + order := []driver.ComputeEnvironmentOrder{{Order: 1, ComputeEnvironment: "ce"}} + if _, err := m.CreateJobQueue(ctx, driver.CreateJobQueueInput{ + Name: "bad", Priority: 1, ComputeEnvironmentOrder: order, + SchedulingPolicyARN: "arn:aws:batch:us-east-1:000000000000:scheduling-policy/none", + }); !errors.IsInvalidArgument(err) { + t.Fatalf("queue with unknown policy: want InvalidArgument, got %v", err) + } + + if _, err := m.CreateJobQueue(ctx, driver.CreateJobQueueInput{ + Name: "fq", Priority: 1, ComputeEnvironmentOrder: order, SchedulingPolicyARN: p.ARN, + }); err != nil { + t.Fatalf("fair-share queue: %v", err) + } + + // The policy can't be removed from the queue, and a FIFO queue can't gain one. + empty := "" + if _, err := m.UpdateJobQueue(ctx, driver.UpdateJobQueueInput{Name: "fq", SchedulingPolicyARN: &empty}); !errors.IsInvalidArgument(err) { + t.Fatalf("removing policy: want InvalidArgument, got %v", err) + } + + if _, err := m.CreateJobQueue(ctx, driver.CreateJobQueueInput{Name: "fifo", Priority: 1, ComputeEnvironmentOrder: order}); err != nil { + t.Fatal(err) + } + + if _, err := m.UpdateJobQueue(ctx, driver.UpdateJobQueueInput{Name: "fifo", SchedulingPolicyARN: &p.ARN}); !errors.IsInvalidArgument(err) { + t.Fatalf("adding policy to FIFO queue: want InvalidArgument, got %v", err) + } + + // Jobs on a fair-share queue need a share identifier. + if _, err := m.RegisterJobDefinition(ctx, driver.RegisterJobDefinitionInput{ + Name: "jd", Type: driver.JDTypeContainer, ContainerProperties: json.RawMessage(`{"image":"x"}`), + }); err != nil { + t.Fatal(err) + } + + if _, err := m.SubmitJob(ctx, driver.SubmitJobInput{JobName: "j", JobQueue: "fq", JobDefinition: "jd"}); !errors.IsInvalidArgument(err) { + t.Fatalf("fair-share submit without shareIdentifier: want InvalidArgument, got %v", err) + } + + if _, err := m.SubmitJob(ctx, driver.SubmitJobInput{JobName: "j", JobQueue: "fq", JobDefinition: "jd", ShareIdentifier: "A1"}); err != nil { + t.Fatalf("fair-share submit: %v", err) + } + + // An in-use policy can't be deleted. + if err := m.DeleteSchedulingPolicy(ctx, p.ARN); !errors.IsFailedPrecondition(err) { + t.Fatalf("delete in-use policy: want FailedPrecondition, got %v", err) + } + + disabled := driver.StateDisabled + if _, err := m.UpdateJobQueue(ctx, driver.UpdateJobQueueInput{Name: "fq", State: &disabled}); err != nil { + t.Fatal(err) + } + + if err := m.DeleteJobQueue(ctx, "fq"); err != nil { + t.Fatal(err) + } + + if err := m.DeleteSchedulingPolicy(ctx, p.ARN); err != nil { + t.Fatalf("delete: %v", err) + } + + if list, _ := m.ListSchedulingPolicies(ctx); len(list) != 0 { + t.Fatalf("policy not deleted: %+v", list) + } + + if err := m.DeleteSchedulingPolicy(ctx, p.ARN); !errors.IsNotFound(err) { + t.Fatalf("delete missing policy: want NotFound, got %v", err) + } +} + +func TestRegisterJobDefinitionTypeValidation(t *testing.T) { + m := newMock(t) + ctx := context.Background() + + cases := map[string]driver.RegisterJobDefinitionInput{ + "unknown type": {Name: "a", Type: "batch", ContainerProperties: json.RawMessage(`{"image":"x"}`)}, + "wrong case": {Name: "a", Type: "CONTAINER", ContainerProperties: json.RawMessage(`{"image":"x"}`)}, + "container without props": {Name: "a", Type: driver.JDTypeContainer}, + "multinode without nodeprop": {Name: "a", Type: driver.JDTypeMultiNode, ContainerProperties: json.RawMessage(`{"image":"x"}`)}, + } + + for name, in := range cases { + if _, err := m.RegisterJobDefinition(ctx, in); !errors.IsInvalidArgument(err) { + t.Errorf("%s: want InvalidArgument, got %v", name, err) + } + } + + if _, err := m.RegisterJobDefinition(ctx, driver.RegisterJobDefinitionInput{ + Name: "mn", Type: driver.JDTypeMultiNode, NodeProperties: json.RawMessage(`{"numNodes":2,"mainNode":0}`), + }); err != nil { + t.Fatalf("valid multinode: %v", err) + } + + if _, err := m.RegisterJobDefinition(ctx, driver.RegisterJobDefinitionInput{ + Name: "eks", Type: driver.JDTypeContainer, EksProperties: json.RawMessage(`{"podProperties":{}}`), + }); err != nil { + t.Fatalf("valid eks container: %v", err) + } +} + +func TestJobsAndPoliciesSurviveSnapshot(t *testing.T) { + src, fc := newAsyncMock(t) + setupQueue(t, src) + ctx := context.Background() + + if _, err := src.CreateSchedulingPolicy(ctx, driver.CreateSchedulingPolicyInput{Name: "sp"}); err != nil { + t.Fatal(err) + } + + j := submit(t, src, driver.SubmitJobInput{Tags: map[string]string{"t": "1"}}) + fc.Advance(time.Second) + + data, err := src.Snapshot(ctx, false) + if err != nil { + t.Fatal(err) + } + + dst := batch.New(config.NewOptions( + config.WithRegion("us-east-1"), config.WithAccountID("000000000000"), + config.WithClock(fc), config.WithAsyncSettle(), + )) + if err := dst.Restore(ctx, data); err != nil { + t.Fatal(err) + } + + if got := describe(t, dst, j.JobID); got.Status != driver.JobStatusRunnable || got.Tags["t"] != "1" { + t.Fatalf("restored job = %+v", got) + } + + fc.Advance(5 * time.Second) + + if got := describe(t, dst, j.JobID); got.Status != driver.JobStatusSucceeded { + t.Fatalf("restored job did not resume its lifecycle: %s", got.Status) + } + + if list, _ := dst.ListSchedulingPolicies(ctx); len(list) != 1 { + t.Fatalf("scheduling policy not restored: %+v", list) + } + + // Sequence numbering continues after restore. + k := submit(t, dst, driver.SubmitJobInput{JobName: "after"}) + + all, _ := listJobs(dst, driver.ListJobsInput{JobQueue: "q", Filters: []driver.JobFilter{{Name: driver.FilterJobDefinition, Values: []string{"jd"}}}}) + if len(all) != 2 || all[0].JobID != k.JobID { + t.Fatalf("post-restore ordering wrong: %+v", all) + } +} diff --git a/providers/aws/batch/scheduling_policy.go b/providers/aws/batch/scheduling_policy.go new file mode 100644 index 000000000..86ae1733b --- /dev/null +++ b/providers/aws/batch/scheduling_policy.go @@ -0,0 +1,281 @@ +package batch + +import ( + "context" + "encoding/json" + "regexp" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/services/batch/driver" +) + +// Fair-share policy bounds from the FairsharePolicy and ShareAttributes API +// reference. +const ( + maxComputeReservation = 99 + maxShareDecaySeconds = 604800 + minWeightFactor = 0.0001 + maxWeightFactor = 999.9999 + maxShareIdentifierLen = 255 + maxDescribeArns = 100 +) + +// resourceNameRE is the CreateSchedulingPolicy / CreateJobQueue name rule: up +// to 128 letters, numbers, hyphens and underscores. +var resourceNameRE = regexp.MustCompile(`^[A-Za-z0-9_-]{1,128}$`) + +// fairsharePolicy is the subset of the FairsharePolicy shape the emulator +// validates; the raw JSON the caller sent is what is stored and echoed. +type fairsharePolicy struct { + ComputeReservation *int64 `json:"computeReservation"` + ShareDecaySeconds *int64 `json:"shareDecaySeconds"` + ShareDistribution []shareAttributes `json:"shareDistribution"` +} + +// shareAttributes is one ShareAttributes entry of a fair-share policy. +type shareAttributes struct { + ShareIdentifier *string `json:"shareIdentifier"` + WeightFactor *float64 `json:"weightFactor"` +} + +//nolint:gocritic // clone intentionally takes a value to return an independent copy-on-write copy +func cloneSchedulingPolicy(p driver.SchedulingPolicy) driver.SchedulingPolicy { + p.FairsharePolicy = copyRaw(p.FairsharePolicy) + p.QuotaSharePolicy = copyRaw(p.QuotaSharePolicy) + p.Tags = copyTags(p.Tags) + + return p +} + +// isSet reports whether a raw JSON member was actually supplied (not absent or +// an explicit null). +func isSet(raw json.RawMessage) bool { + return len(raw) > 0 && string(raw) != "null" +} + +// validateFairshare enforces the documented FairsharePolicy bounds. +func validateFairshare(raw json.RawMessage) error { + if !isSet(raw) { + return nil + } + + var fp fairsharePolicy + if err := json.Unmarshal(raw, &fp); err != nil { + return cerrors.Newf(cerrors.InvalidArgument, "invalid fairsharePolicy: %v", err) + } + + if err := checkRange("computeReservation", fp.ComputeReservation, maxComputeReservation); err != nil { + return err + } + + if err := checkRange("shareDecaySeconds", fp.ShareDecaySeconds, maxShareDecaySeconds); err != nil { + return err + } + + for i := range fp.ShareDistribution { + if err := validateShareAttributes(i, &fp.ShareDistribution[i]); err != nil { + return err + } + } + + return nil +} + +// checkRange rejects a set fair-share integer outside [0, maxValue]. +func checkRange(field string, v *int64, maxValue int64) error { + if v != nil && (*v < 0 || *v > maxValue) { + return cerrors.Newf(cerrors.InvalidArgument, + "fairsharePolicy.%s must be between 0 and %d, got %d", field, maxValue, *v) + } + + return nil +} + +// validateShareAttributes enforces the ShareAttributes bounds of one +// shareDistribution entry: a required identifier of at most 255 characters +// (plus an optional trailing '*') and a weight factor in [0.0001, 999.9999]. +func validateShareAttributes(i int, sd *shareAttributes) error { + if sd.ShareIdentifier == nil || *sd.ShareIdentifier == "" { + return cerrors.Newf(cerrors.InvalidArgument, "fairsharePolicy.shareDistribution[%d].shareIdentifier is required", i) + } + + if len(*sd.ShareIdentifier) > maxShareIdentifierLen+1 { + return cerrors.Newf(cerrors.InvalidArgument, + "fairsharePolicy.shareDistribution[%d].shareIdentifier is longer than %d characters", i, maxShareIdentifierLen) + } + + if w := sd.WeightFactor; w != nil && (*w < minWeightFactor || *w > maxWeightFactor) { + return cerrors.Newf(cerrors.InvalidArgument, + "fairsharePolicy.shareDistribution[%d].weightFactor must be between %v and %v, got %v", + i, minWeightFactor, maxWeightFactor, *w) + } + + return nil +} + +// CreateSchedulingPolicy creates a fair-share (or quota-share) scheduling +// policy. Only one of the two policy kinds may be set. +func (m *Mock) CreateSchedulingPolicy( + _ context.Context, in driver.CreateSchedulingPolicyInput, +) (*driver.SchedulingPolicy, error) { + if in.Name == "" { + return nil, cerrors.New(cerrors.InvalidArgument, "name is required") + } + + if !resourceNameRE.MatchString(in.Name) { + return nil, cerrors.Newf(cerrors.InvalidArgument, + "scheduling policy name %q must be up to 128 letters, numbers, hyphens and underscores", in.Name) + } + + if isSet(in.FairsharePolicy) && isSet(in.QuotaSharePolicy) { + return nil, cerrors.New(cerrors.InvalidArgument, "only one of fairsharePolicy or quotaSharePolicy can be set") + } + + if err := validateTags(in.Tags); err != nil { + return nil, err + } + + if err := validateFairshare(in.FairsharePolicy); err != nil { + return nil, err + } + + p := driver.SchedulingPolicy{ + Name: in.Name, + ARN: m.arn(kindSchedulingPolicy, in.Name), + FairsharePolicy: copyRaw(nonNull(in.FairsharePolicy)), + QuotaSharePolicy: copyRaw(nonNull(in.QuotaSharePolicy)), + Tags: copyTags(in.Tags), + } + + if !m.schedPolicies.SetIfAbsent(in.Name, p) { + return nil, cerrors.Newf(cerrors.AlreadyExists, "scheduling policy %q already exists", in.Name) + } + + out := cloneSchedulingPolicy(p) + + return &out, nil +} + +// nonNull drops an explicit JSON null so it is stored as "absent". +func nonNull(raw json.RawMessage) json.RawMessage { + if !isSet(raw) { + return nil + } + + return raw +} + +// DescribeSchedulingPolicies returns the named policies (ARN or name) in +// request order; unknown entries are omitted. At least one and at most 100 +// ARNs are required. +func (m *Mock) DescribeSchedulingPolicies(_ context.Context, arns []string) ([]driver.SchedulingPolicy, error) { + if len(arns) == 0 { + return nil, cerrors.New(cerrors.InvalidArgument, "arns is required") + } + + if len(arns) > maxDescribeArns { + return nil, cerrors.Newf(cerrors.InvalidArgument, "arns accepts at most %d entries, got %d", maxDescribeArns, len(arns)) + } + + return describeByNames(m.schedPolicies, arns, kindSchedulingPolicy, cloneSchedulingPolicy), nil +} + +// ListSchedulingPolicies returns every policy, ordered by name. +func (m *Mock) ListSchedulingPolicies(_ context.Context) ([]driver.SchedulingPolicy, error) { + return describeByNames(m.schedPolicies, nil, kindSchedulingPolicy, cloneSchedulingPolicy), nil +} + +// UpdateSchedulingPolicy replaces the policy details. A fair-share policy can't +// be turned into a quota-share policy or vice versa. +func (m *Mock) UpdateSchedulingPolicy(_ context.Context, in driver.UpdateSchedulingPolicyInput) error { + if in.ARN == "" { + return cerrors.New(cerrors.InvalidArgument, "arn is required") + } + + if isSet(in.FairsharePolicy) && isSet(in.QuotaSharePolicy) { + return cerrors.New(cerrors.InvalidArgument, "only one of fairsharePolicy or quotaSharePolicy can be set") + } + + if err := validateFairshare(in.FairsharePolicy); err != nil { + return err + } + + name := nameFromARN(kindSchedulingPolicy, in.ARN) + + cur, ok := m.schedPolicies.Get(name) + if !ok { + return cerrors.Newf(cerrors.NotFound, "scheduling policy %q does not exist", in.ARN) + } + + if err := checkPolicyKind(&cur, &in); err != nil { + return err + } + + m.schedPolicies.Update(name, func(p driver.SchedulingPolicy) driver.SchedulingPolicy { + next := cloneSchedulingPolicy(p) + if isSet(in.FairsharePolicy) { + next.FairsharePolicy = copyRaw(in.FairsharePolicy) + } + + if isSet(in.QuotaSharePolicy) { + next.QuotaSharePolicy = copyRaw(in.QuotaSharePolicy) + } + + return next + }) + + return nil +} + +// checkPolicyKind rejects an update that would turn a fair-share policy into a +// quota-share one or vice versa. +func checkPolicyKind(cur *driver.SchedulingPolicy, in *driver.UpdateSchedulingPolicyInput) error { + if isSet(in.FairsharePolicy) && isSet(cur.QuotaSharePolicy) { + return cerrors.New(cerrors.InvalidArgument, "a quotaSharePolicy can't be changed to a fairsharePolicy") + } + + if isSet(in.QuotaSharePolicy) && isSet(cur.FairsharePolicy) { + return cerrors.New(cerrors.InvalidArgument, "a fairsharePolicy can't be changed to a quotaSharePolicy") + } + + return nil +} + +// DeleteSchedulingPolicy deletes a policy. A policy still referenced by a job +// queue can't be deleted. +func (m *Mock) DeleteSchedulingPolicy(_ context.Context, arn string) error { + if arn == "" { + return cerrors.New(cerrors.InvalidArgument, "arn is required") + } + + name := nameFromARN(kindSchedulingPolicy, arn) + + p, ok := m.schedPolicies.Get(name) + if !ok { + return cerrors.Newf(cerrors.NotFound, "scheduling policy %q does not exist", arn) + } + + queues := m.jobQueues.SortedValues() + for i := range queues { + q := &queues[i] + if q.SchedulingPolicyARN != "" && nameFromARN(kindSchedulingPolicy, q.SchedulingPolicyARN) == name { + return cerrors.Newf(cerrors.FailedPrecondition, + "scheduling policy %q is in use by job queue %q", p.ARN, q.Name) + } + } + + m.schedPolicies.Delete(name) + + return nil +} + +// resolveSchedulingPolicy returns the canonical ARN of an existing policy, or a +// ClientException-mapped error when it does not exist. +func (m *Mock) resolveSchedulingPolicy(ref string) (string, error) { + p, ok := m.schedPolicies.Get(nameFromARN(kindSchedulingPolicy, ref)) + if !ok { + return "", cerrors.Newf(cerrors.InvalidArgument, "scheduling policy %q does not exist", ref) + } + + return p.ARN, nil +} diff --git a/providers/aws/batch/snapshot.go b/providers/aws/batch/snapshot.go index 1bc5a8265..6fbfa1e86 100644 --- a/providers/aws/batch/snapshot.go +++ b/providers/aws/batch/snapshot.go @@ -13,13 +13,18 @@ var _ snapshot.Snapshottable = (*Mock)(nil) // batchSnapshot is the full serialized state of the Batch mock. Every store // holds a plain JSON-serializable value, so each round-trips through the generic // memstore dump helper under its original keys (compute-environment/job-queue -// name, job-definition "name:revision"). The registration mutex and the wired +// name, job-definition "name:revision", scheduling-policy name, job ID). Job +// records carry their submission instant and phase duration, so a restored job +// resumes its lifecycle where it left off. The registration mutex and the wired // *config.Options are intentionally not serialized. type batchSnapshot struct { - ComputeEnvs json.RawMessage `json:"computeEnvs,omitempty"` - JobQueues json.RawMessage `json:"jobQueues,omitempty"` - JobDefs json.RawMessage `json:"jobDefs,omitempty"` - JobDefMaxRev json.RawMessage `json:"jobDefMaxRev,omitempty"` + ComputeEnvs json.RawMessage `json:"computeEnvs,omitempty"` + JobQueues json.RawMessage `json:"jobQueues,omitempty"` + JobDefs json.RawMessage `json:"jobDefs,omitempty"` + JobDefMaxRev json.RawMessage `json:"jobDefMaxRev,omitempty"` + SchedPolicies json.RawMessage `json:"schedulingPolicies,omitempty"` + Jobs json.RawMessage `json:"jobs,omitempty"` + JobSeq json.RawMessage `json:"jobSeq,omitempty"` } // Snapshot captures the mock's entire state as JSON. includeAssets is unused. Batch holds no bulk @@ -35,6 +40,9 @@ func (m *Mock) Snapshot(_ context.Context, _ bool) (json.RawMessage, error) { {&snap.JobQueues, m.jobQueues.Snapshot}, {&snap.JobDefs, m.jobDefs.Snapshot}, {&snap.JobDefMaxRev, m.jobDefMaxRev.Snapshot}, + {&snap.SchedPolicies, m.schedPolicies.Snapshot}, + {&snap.Jobs, m.jobs.Snapshot}, + {&snap.JobSeq, m.jobSeq.Snapshot}, } for _, d := range dumps { @@ -64,6 +72,9 @@ func (m *Mock) Restore(_ context.Context, data json.RawMessage) error { {snap.JobQueues, m.jobQueues.LoadSnapshot}, {snap.JobDefs, m.jobDefs.LoadSnapshot}, {snap.JobDefMaxRev, m.jobDefMaxRev.LoadSnapshot}, + {snap.SchedPolicies, m.schedPolicies.LoadSnapshot}, + {snap.Jobs, m.jobs.LoadSnapshot}, + {snap.JobSeq, m.jobSeq.LoadSnapshot}, } for _, l := range loads { diff --git a/providers/aws/batch/tags.go b/providers/aws/batch/tags.go index 76b03b983..8ba8ccdef 100644 --- a/providers/aws/batch/tags.go +++ b/providers/aws/batch/tags.go @@ -17,6 +17,10 @@ func kindFromARN(arn string) string { return kindJobQueue case strings.Contains(arn, ":"+kindJobDefinition+"/"): return kindJobDefinition + case strings.Contains(arn, ":"+kindSchedulingPolicy+"/"): + return kindSchedulingPolicy + case strings.Contains(arn, ":"+kindJob+"/"): + return kindJob default: return "" } @@ -44,11 +48,64 @@ func removeTags(existing map[string]string, keys []string) map[string]string { return out } -// TagResource adds or replaces tags on the resource named by arn. +// Tag limits from the Batch TagResource reference: at most 50 tags, keys of +// 1-128 characters, values of at most 256 characters. The "aws:" key prefix is +// reserved for AWS (see "Tagging AWS resources"). +const ( + maxTags = 50 + maxTagKeyLen = 128 + maxTagValueLen = 256 + reservedPrefix = "aws:" +) + +// validateTags checks one tag map against the TagResource limits. +func validateTags(tags map[string]string) error { + if len(tags) > maxTags { + return cerrors.Newf(cerrors.InvalidArgument, "a resource can have at most %d tags, got %d", maxTags, len(tags)) + } + + for k, v := range tags { + switch { + case k == "" || len(k) > maxTagKeyLen: + return cerrors.Newf(cerrors.InvalidArgument, "tag key %q must be 1 to %d characters", k, maxTagKeyLen) + case len(v) > maxTagValueLen: + return cerrors.Newf(cerrors.InvalidArgument, "tag value for key %q must be at most %d characters", k, maxTagValueLen) + case strings.HasPrefix(strings.ToLower(k), reservedPrefix): + return cerrors.Newf(cerrors.InvalidArgument, "tag key %q uses the reserved prefix %q", k, reservedPrefix) + } + } + + return nil +} + +// TagResource adds or replaces tags on the resource named by arn. Child jobs of +// an array job can't be tagged. func (m *Mock) TagResource(_ context.Context, arn string, tags map[string]string) error { - return m.mutateTags(arn, func(existing map[string]string) map[string]string { - return mergeTags(existing, tags) + if len(tags) == 0 { + return cerrors.New(cerrors.InvalidArgument, "tags is required") + } + + if err := validateTags(tags); err != nil { + return err + } + + var limitErr error + + err := m.mutateTags(arn, func(existing map[string]string) map[string]string { + merged := mergeTags(existing, tags) + if len(merged) > maxTags { + limitErr = cerrors.Newf(cerrors.InvalidArgument, "a resource can have at most %d tags", maxTags) + + return existing + } + + return merged }) + if err != nil { + return err + } + + return limitErr } // UntagResource removes the given tag keys from the resource named by arn. @@ -60,24 +117,45 @@ func (m *Mock) UntagResource(_ context.Context, arn string, keys []string) error // ListTagsForResource returns the tags on the resource named by arn. func (m *Mock) ListTagsForResource(_ context.Context, arn string) (map[string]string, error) { - switch kindFromARN(arn) { + kind := kindFromARN(arn) + if kind == "" { + return nil, cerrors.Newf(cerrors.InvalidArgument, "unsupported resource ARN %q", arn) + } + + tags, ok := m.tagsOf(kind, nameFromARN(kind, arn)) + if !ok { + return nil, cerrors.Newf(cerrors.NotFound, "resource %q does not exist", arn) + } + + return copyTags(tags), nil +} + +// tagsOf returns the stored tags of the named resource of the given kind. +func (m *Mock) tagsOf(kind, name string) (map[string]string, bool) { + switch kind { case kindComputeEnvironment: - if ce, ok := m.computeEnvs.Get(nameFromARN(kindComputeEnvironment, arn)); ok { - return copyTags(ce.Tags), nil - } + ce, ok := m.computeEnvs.Get(name) + + return ce.Tags, ok case kindJobQueue: - if q, ok := m.jobQueues.Get(nameFromARN(kindJobQueue, arn)); ok { - return copyTags(q.Tags), nil - } + q, ok := m.jobQueues.Get(name) + + return q.Tags, ok case kindJobDefinition: - if jd, ok := m.jobDefs.Get(nameFromARN(kindJobDefinition, arn)); ok { - return copyTags(jd.Tags), nil - } + jd, ok := m.jobDefs.Get(name) + + return jd.Tags, ok + case kindSchedulingPolicy: + p, ok := m.schedPolicies.Get(name) + + return p.Tags, ok + case kindJob: + r, ok := m.jobs.Get(name) + + return r.Job.Tags, ok default: - return nil, cerrors.Newf(cerrors.InvalidArgument, "unsupported resource ARN %q", arn) + return nil, false } - - return nil, cerrors.Newf(cerrors.NotFound, "resource %q does not exist", arn) } // mutateTags applies fn to the tag map of the resource named by arn. @@ -109,6 +187,24 @@ func (m *Mock) mutateTags(arn string, fn func(map[string]string) map[string]stri return next }) + case kindSchedulingPolicy: + ok = m.schedPolicies.Update(nameFromARN(kindSchedulingPolicy, arn), + func(p driver.SchedulingPolicy) driver.SchedulingPolicy { + next := cloneSchedulingPolicy(p) + next.Tags = fn(next.Tags) + + return next + }) + case kindJob: + // jobsMu: publishJobEvents rewrites job records under it. + m.jobsMu.Lock() + ok = m.jobs.Update(nameFromARN(kindJob, arn), func(r jobRecord) jobRecord { + next := cloneJobRecord(r) + next.Job.Tags = fn(next.Job.Tags) + + return next + }) + m.jobsMu.Unlock() default: return cerrors.Newf(cerrors.InvalidArgument, "unsupported resource ARN %q", arn) } diff --git a/providers/aws/tickables_test.go b/providers/aws/tickables_test.go index 2347f0c1a..ab76f1d5e 100644 --- a/providers/aws/tickables_test.go +++ b/providers/aws/tickables_test.go @@ -8,17 +8,17 @@ import ( ) // TestTickablesRegistersTimeDrivenServices checks the serve ticker reaches the -// region's own SQS, EFS, CloudWatch, SSM and CloudFormation, so queue gauges, -// EFS metrics, due alarms, parameter policies and settling stack operations run -// without a read. SQS and EFS come before CloudWatch so alarms see the metrics -// published in the same tick. +// region's own SQS, EFS, CloudWatch, SSM, CloudFormation and Batch, so queue +// gauges, EFS metrics, due alarms, parameter policies, settling stack operations +// and Batch job state-change events run without a read. SQS and EFS come before +// CloudWatch so alarms see the metrics published in the same tick. func TestTickablesRegistersTimeDrivenServices(t *testing.T) { p := aws.New() got := p.Tickables() - if len(got) != 5 || got[0] != config.Tickable(p.SQS) || got[1] != config.Tickable(p.EFS) || + if len(got) != 6 || got[0] != config.Tickable(p.SQS) || got[1] != config.Tickable(p.EFS) || got[2] != config.Tickable(p.CloudWatch) || got[3] != config.Tickable(p.SSM) || - got[4] != config.Tickable(p.CloudFormation) { - t.Fatalf("Tickables() = %v, want this provider's SQS, EFS, CloudWatch, SSM and CloudFormation", got) + got[4] != config.Tickable(p.CloudFormation) || got[5] != config.Tickable(p.Batch) { + t.Fatalf("Tickables() = %v, want this provider's SQS, EFS, CloudWatch, SSM, CloudFormation and Batch", got) } } diff --git a/server/aws/batch/handler.go b/server/aws/batch/handler.go index fcced33a4..2883e96bb 100644 --- a/server/aws/batch/handler.go +++ b/server/aws/batch/handler.go @@ -2,8 +2,8 @@ // awsRestjson1) as a server.Handler. Point the real // aws-sdk-go-v2/service/batch client (or the `aws batch` CLI, or the // aws_batch_* Terraform resources) at a Server registered with this handler and -// the compute-environment, job-queue, and job-definition operations work -// end-to-end against an in-memory driver. +// the compute-environment, job-queue, job-definition, scheduling-policy and job +// operations work end-to-end against an in-memory driver. // // Batch routes by HTTP verb + a fixed operation path under the /v1/ prefix // (e.g. POST /v1/createcomputeenvironment); there is no X-Amz-Target header. @@ -35,6 +35,16 @@ const ( opRegisterJobDefinition = "registerjobdefinition" opDescribeJobDefinitions = "describejobdefinitions" opDeregisterJobDefinition = "deregisterjobdefinition" + opCreateSchedulingPolicy = "createschedulingpolicy" + opDescribeSchedulingPolicies = "describeschedulingpolicies" + opListSchedulingPolicies = "listschedulingpolicies" + opUpdateSchedulingPolicy = "updateschedulingpolicy" + opDeleteSchedulingPolicy = "deleteschedulingpolicy" + opSubmitJob = "submitjob" + opDescribeJobs = "describejobs" + opListJobs = "listjobs" + opCancelJob = "canceljob" + opTerminateJob = "terminatejob" opTags = "tags" ) @@ -63,6 +73,16 @@ var knownOps = map[string]bool{ opRegisterJobDefinition: true, opDescribeJobDefinitions: true, opDeregisterJobDefinition: true, + opCreateSchedulingPolicy: true, + opDescribeSchedulingPolicies: true, + opListSchedulingPolicies: true, + opUpdateSchedulingPolicy: true, + opDeleteSchedulingPolicy: true, + opSubmitJob: true, + opDescribeJobs: true, + opListJobs: true, + opCancelJob: true, + opTerminateJob: true, opTags: true, } @@ -118,7 +138,9 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (h *Handler) serveOperation(w http.ResponseWriter, r *http.Request, op string) { if h.serveComputeEnvironmentOp(w, r, op) || h.serveJobQueueOp(w, r, op) || - h.serveJobDefinitionOp(w, r, op) { + h.serveJobDefinitionOp(w, r, op) || + h.serveSchedulingPolicyOp(w, r, op) || + h.serveJobOp(w, r, op) { return } @@ -180,6 +202,48 @@ func (h *Handler) serveJobDefinitionOp(w http.ResponseWriter, r *http.Request, o return true } +// serveSchedulingPolicyOp handles the scheduling-policy operations, or returns +// false if op is not one of them. +func (h *Handler) serveSchedulingPolicyOp(w http.ResponseWriter, r *http.Request, op string) bool { + switch op { + case opCreateSchedulingPolicy: + h.createSchedulingPolicy(w, r) + case opDescribeSchedulingPolicies: + h.describeSchedulingPolicies(w, r) + case opListSchedulingPolicies: + h.listSchedulingPolicies(w, r) + case opUpdateSchedulingPolicy: + h.updateSchedulingPolicy(w, r) + case opDeleteSchedulingPolicy: + h.deleteSchedulingPolicy(w, r) + default: + return false + } + + return true +} + +// serveJobOp handles the job operations, or returns false if op is not one of +// them. +func (h *Handler) serveJobOp(w http.ResponseWriter, r *http.Request, op string) bool { + switch op { + case opSubmitJob: + h.submitJob(w, r) + case opDescribeJobs: + h.describeJobs(w, r) + case opListJobs: + h.listJobs(w, r) + case opCancelJob: + h.cancelJob(w, r) + case opTerminateJob: + h.terminateJob(w, r) + default: + return false + } + + return true +} + // splitPath splits a URL path into its non-empty segments. func splitPath(p string) []string { p = strings.Trim(p, "/") diff --git a/server/aws/batch/jobs.go b/server/aws/batch/jobs.go new file mode 100644 index 000000000..e7b67ccdb --- /dev/null +++ b/server/aws/batch/jobs.go @@ -0,0 +1,119 @@ +package batch + +import ( + "context" + "net/http" + + "github.com/stackshy/cloudemu/v2/services/batch/driver" +) + +func (h *Handler) submitJob(w http.ResponseWriter, r *http.Request) { + var req submitJobRequest + if !decodeJSON(w, r, &req) { + return + } + + in := driver.SubmitJobInput{ + JobName: req.JobName, + JobQueue: req.JobQueue, + JobDefinition: req.JobDefinition, + DependsOn: depsToDriver(req.DependsOn), + Parameters: req.Parameters, + ContainerOverrides: req.ContainerOverrides, + NodeOverrides: req.NodeOverrides, + EcsPropertiesOverride: req.EcsPropertiesOverride, + EksPropertiesOverride: req.EksPropertiesOverride, + RetryStrategy: req.RetryStrategy, + Timeout: req.Timeout, + PropagateTags: req.PropagateTags, + SchedulingPriorityOverride: req.SchedulingPriorityOverride, + ShareIdentifier: req.ShareIdentifier, + Tags: req.Tags, + } + + if req.ArrayProperties != nil && req.ArrayProperties.Size != nil { + in.ArraySize = req.ArrayProperties.Size + } + + job, err := h.batch.SubmitJob(r.Context(), in) + if err != nil { + writeErr(w, err) + + return + } + + writeJSON(w, submitJobResponse{JobArn: job.JobARN, JobID: job.JobID, JobName: job.JobName}) +} + +func (h *Handler) describeJobs(w http.ResponseWriter, r *http.Request) { + var req describeJobsRequest + if !decodeJSON(w, r, &req) { + return + } + + jobs, err := h.batch.DescribeJobs(r.Context(), req.Jobs) + if err != nil { + writeErr(w, err) + + return + } + + details := make([]jobDetail, 0, len(jobs)) + for i := range jobs { + details = append(details, toJobDetail(&jobs[i])) + } + + writeJSON(w, describeJobsResponse{Jobs: details}) +} + +func (h *Handler) listJobs(w http.ResponseWriter, r *http.Request) { + var req listJobsRequest + if !decodeJSON(w, r, &req) { + return + } + + page, err := h.batch.ListJobs(r.Context(), driver.ListJobsInput{ + JobQueue: req.JobQueue, + ArrayJobID: req.ArrayJobID, + MultiNodeJobID: req.MultiNodeJobID, + JobStatus: req.JobStatus, + Filters: filtersToDriver(req.Filters), + MaxResults: req.MaxResults, + NextToken: req.NextToken, + }) + if err != nil { + writeErr(w, err) + + return + } + + out := make([]jobSummary, 0, len(page.Jobs)) + for i := range page.Jobs { + out = append(out, toJobSummary(&page.Jobs[i])) + } + + writeJSON(w, listJobsResponse{JobSummaryList: out, NextToken: page.NextToken}) +} + +func (h *Handler) cancelJob(w http.ResponseWriter, r *http.Request) { + h.stopJob(w, r, h.batch.CancelJob) +} + +func (h *Handler) terminateJob(w http.ResponseWriter, r *http.Request) { + h.stopJob(w, r, h.batch.TerminateJob) +} + +func (*Handler) stopJob(w http.ResponseWriter, r *http.Request, stop func(ctx context.Context, id, reason string) error) { + var req stopJobRequest + if !decodeJSON(w, r, &req) { + return + } + + if err := stop(r.Context(), req.JobID, req.Reason); err != nil { + writeErr(w, err) + + return + } + + writeJSON(w, emptyResponse{}) +} diff --git a/server/aws/batch/jobs_paging_sdk_test.go b/server/aws/batch/jobs_paging_sdk_test.go new file mode 100644 index 000000000..d81c44794 --- /dev/null +++ b/server/aws/batch/jobs_paging_sdk_test.go @@ -0,0 +1,96 @@ +package batch_test + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awsbatch "github.com/aws/aws-sdk-go-v2/service/batch" + batchtypes "github.com/aws/aws-sdk-go-v2/service/batch/types" +) + +func TestSDKListJobsPaginatorAndFilters(t *testing.T) { + ctx := context.Background() + c := newBatchClient(t) + setupSDKQueue(t, c, nil) + + for _, n := range []string{"alpha", "beta", "gamma"} { + if _, err := c.SubmitJob(ctx, &awsbatch.SubmitJobInput{ + JobName: aws.String(n), JobQueue: aws.String("q"), JobDefinition: aws.String("jd"), + }); err != nil { + t.Fatalf("SubmitJob: %v", err) + } + } + + p := awsbatch.NewListJobsPaginator(c, &awsbatch.ListJobsInput{ + JobQueue: aws.String("q"), JobStatus: batchtypes.JobStatusSucceeded, MaxResults: aws.Int32(2), + }) + + var names []string + + pages := 0 + + for p.HasMorePages() { + page, err := p.NextPage(ctx) + if err != nil { + t.Fatalf("NextPage: %v", err) + } + + pages++ + + for _, s := range page.JobSummaryList { + names = append(names, aws.ToString(s.JobName)) + } + } + + if pages != 2 || len(names) != 3 || names[0] != "gamma" || names[2] != "alpha" { + t.Fatalf("paginated ListJobs = %v over %d pages, want [gamma beta alpha] over 2", names, pages) + } + + byName, err := c.ListJobs(ctx, &awsbatch.ListJobsInput{ + JobQueue: aws.String("q"), + Filters: []batchtypes.KeyValuesPair{{Name: aws.String("JOB_NAME"), Values: []string{"GAM*"}}}, + }) + if err != nil || len(byName.JobSummaryList) != 1 || aws.ToString(byName.JobSummaryList[0].JobName) != "gamma" { + t.Fatalf("JOB_NAME filter: %+v, %v", byName, err) + } + + _, err = c.ListJobs(ctx, &awsbatch.ListJobsInput{JobQueue: aws.String("q"), NextToken: aws.String("!!")}) + requireClientException(t, err, "ListJobs bad nextToken") + + _, err = c.ListJobs(ctx, &awsbatch.ListJobsInput{}) + requireClientException(t, err, "ListJobs without jobQueue/arrayJobId/multiNodeJobId") +} + +func TestSDKReservedTagPrefixRejected(t *testing.T) { + ctx := context.Background() + c := newBatchClient(t) + setupSDKQueue(t, c, nil) + + _, err := c.SubmitJob(ctx, &awsbatch.SubmitJobInput{ + JobName: aws.String("j"), JobQueue: aws.String("q"), JobDefinition: aws.String("jd"), + Tags: map[string]string{"aws:team": "x"}, + }) + requireClientException(t, err, "SubmitJob with aws: tag") + + sub, err := c.SubmitJob(ctx, &awsbatch.SubmitJobInput{ + JobName: aws.String("j"), JobQueue: aws.String("q"), JobDefinition: aws.String("jd"), + }) + if err != nil { + t.Fatalf("SubmitJob: %v", err) + } + + _, err = c.TagResource(ctx, &awsbatch.TagResourceInput{ResourceArn: sub.JobArn, Tags: map[string]string{"AWS:x": "y"}}) + requireClientException(t, err, "TagResource with aws: tag") + + if _, err := c.TagResource(ctx, &awsbatch.TagResourceInput{ + ResourceArn: sub.JobArn, Tags: map[string]string{"team": "a"}, + }); err != nil { + t.Fatalf("TagResource(job): %v", err) + } + + tags, err := c.ListTagsForResource(ctx, &awsbatch.ListTagsForResourceInput{ResourceArn: sub.JobArn}) + if err != nil || tags.Tags["team"] != "a" { + t.Fatalf("ListTagsForResource(job): %+v, %v", tags, err) + } +} diff --git a/server/aws/batch/jobs_sdk_test.go b/server/aws/batch/jobs_sdk_test.go new file mode 100644 index 000000000..778c1d327 --- /dev/null +++ b/server/aws/batch/jobs_sdk_test.go @@ -0,0 +1,240 @@ +package batch_test + +import ( + "context" + "errors" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awsbatch "github.com/aws/aws-sdk-go-v2/service/batch" + batchtypes "github.com/aws/aws-sdk-go-v2/service/batch/types" +) + +// setupSDKQueue creates a Fargate CE, a queue "q" (optionally fair-share) and a +// container job definition "jd". +func setupSDKQueue(t *testing.T, c *awsbatch.Client, policyArn *string) { + t.Helper() + + ctx := context.Background() + createFargateCE(t, c, "ce") + + if _, err := c.CreateJobQueue(ctx, &awsbatch.CreateJobQueueInput{ + JobQueueName: aws.String("q"), + Priority: aws.Int32(1), + SchedulingPolicyArn: policyArn, + ComputeEnvironmentOrder: []batchtypes.ComputeEnvironmentOrder{ + {Order: aws.Int32(1), ComputeEnvironment: aws.String("ce")}, + }, + }); err != nil { + t.Fatalf("CreateJobQueue: %v", err) + } + + if _, err := c.RegisterJobDefinition(ctx, &awsbatch.RegisterJobDefinitionInput{ + JobDefinitionName: aws.String("jd"), + Type: batchtypes.JobDefinitionTypeContainer, + ContainerProperties: &batchtypes.ContainerProperties{ + Image: aws.String("busybox"), + Command: []string{"echo", "hi"}, + }, + }); err != nil { + t.Fatalf("RegisterJobDefinition: %v", err) + } +} + +func requireClientException(t *testing.T, err error, what string) { + t.Helper() + + var ce *batchtypes.ClientException + if !errors.As(err, &ce) { + t.Fatalf("%s: want ClientException, got %v", what, err) + } +} + +func TestSDKJobLifecycle(t *testing.T) { + ctx := context.Background() + c := newBatchClient(t) + setupSDKQueue(t, c, nil) + + sub, err := c.SubmitJob(ctx, &awsbatch.SubmitJobInput{ + JobName: aws.String("hello"), + JobQueue: aws.String("q"), + JobDefinition: aws.String("jd"), + ContainerOverrides: &batchtypes.ContainerOverrides{Command: []string{"echo", "bye"}}, + Tags: map[string]string{"team": "a"}, + }) + if err != nil { + t.Fatalf("SubmitJob: %v", err) + } + + if aws.ToString(sub.JobId) == "" || aws.ToString(sub.JobArn) == "" || aws.ToString(sub.JobName) != "hello" { + t.Fatalf("SubmitJob response: %+v", sub) + } + + desc, err := c.DescribeJobs(ctx, &awsbatch.DescribeJobsInput{Jobs: []string{aws.ToString(sub.JobId), "unknown"}}) + if err != nil { + t.Fatalf("DescribeJobs: %v", err) + } + + if len(desc.Jobs) != 1 { + t.Fatalf("want 1 job, got %d", len(desc.Jobs)) + } + + j := desc.Jobs[0] + if j.Status != batchtypes.JobStatusSucceeded || j.Container == nil || aws.ToInt32(j.Container.ExitCode) != 0 || + j.Container.Command[1] != "bye" || aws.ToInt64(j.CreatedAt) == 0 || aws.ToInt64(j.StoppedAt) == 0 || + j.Tags["team"] != "a" { + t.Fatalf("DescribeJobs detail: %+v", j) + } + + list, err := c.ListJobs(ctx, &awsbatch.ListJobsInput{JobQueue: aws.String("q"), JobStatus: batchtypes.JobStatusSucceeded}) + if err != nil || len(list.JobSummaryList) != 1 || aws.ToString(list.JobSummaryList[0].JobId) != aws.ToString(sub.JobId) { + t.Fatalf("ListJobs: %+v, %v", list, err) + } + + // Default ListJobs lists RUNNING jobs only. + running, err := c.ListJobs(ctx, &awsbatch.ListJobsInput{JobQueue: aws.String("q")}) + if err != nil || len(running.JobSummaryList) != 0 { + t.Fatalf("default ListJobs: %+v, %v", running, err) + } + + // Cancel/Terminate of a finished job succeed with no effect; unknown jobs fail. + if _, err := c.TerminateJob(ctx, &awsbatch.TerminateJobInput{JobId: sub.JobId, Reason: aws.String("r")}); err != nil { + t.Fatalf("TerminateJob: %v", err) + } + + _, err = c.CancelJob(ctx, &awsbatch.CancelJobInput{JobId: aws.String("missing"), Reason: aws.String("r")}) + requireClientException(t, err, "CancelJob unknown job") + + _, err = c.SubmitJob(ctx, &awsbatch.SubmitJobInput{ + JobName: aws.String("x"), JobQueue: aws.String("nope"), JobDefinition: aws.String("jd"), + }) + requireClientException(t, err, "SubmitJob unknown queue") + + _, err = c.SubmitJob(ctx, &awsbatch.SubmitJobInput{ + JobName: aws.String("x"), JobQueue: aws.String("q"), JobDefinition: aws.String("jd"), + ArrayProperties: &batchtypes.ArrayProperties{Size: aws.Int32(1)}, + }) + requireClientException(t, err, "SubmitJob array size 1") +} + +func TestSDKArrayJob(t *testing.T) { + ctx := context.Background() + c := newBatchClient(t) + setupSDKQueue(t, c, nil) + + sub, err := c.SubmitJob(ctx, &awsbatch.SubmitJobInput{ + JobName: aws.String("arr"), JobQueue: aws.String("q"), JobDefinition: aws.String("jd"), + ArrayProperties: &batchtypes.ArrayProperties{Size: aws.Int32(3)}, + }) + if err != nil { + t.Fatalf("SubmitJob: %v", err) + } + + desc, err := c.DescribeJobs(ctx, &awsbatch.DescribeJobsInput{Jobs: []string{aws.ToString(sub.JobId), aws.ToString(sub.JobId) + ":2"}}) + if err != nil || len(desc.Jobs) != 2 { + t.Fatalf("DescribeJobs: %+v, %v", desc, err) + } + + parent, child := desc.Jobs[0], desc.Jobs[1] + if aws.ToInt32(parent.ArrayProperties.Size) != 3 || parent.ArrayProperties.StatusSummary["SUCCEEDED"] != 3 { + t.Fatalf("parent arrayProperties: %+v", parent.ArrayProperties) + } + + if aws.ToInt32(child.ArrayProperties.Index) != 2 { + t.Fatalf("child arrayProperties: %+v", child.ArrayProperties) + } + + kids, err := c.ListJobs(ctx, &awsbatch.ListJobsInput{ArrayJobId: sub.JobId, JobStatus: batchtypes.JobStatusSucceeded}) + if err != nil || len(kids.JobSummaryList) != 3 { + t.Fatalf("ListJobs children: %+v, %v", kids, err) + } +} + +func TestSDKSchedulingPolicies(t *testing.T) { + ctx := context.Background() + c := newBatchClient(t) + + created, err := c.CreateSchedulingPolicy(ctx, &awsbatch.CreateSchedulingPolicyInput{ + Name: aws.String("fair"), + FairsharePolicy: &batchtypes.FairsharePolicy{ + ShareDecaySeconds: aws.Int32(3600), + ComputeReservation: aws.Int32(1), + ShareDistribution: []batchtypes.ShareAttributes{ + {ShareIdentifier: aws.String("A*"), WeightFactor: aws.Float32(0.5)}, + }, + }, + Tags: map[string]string{"k": "v"}, + }) + if err != nil { + t.Fatalf("CreateSchedulingPolicy: %v", err) + } + + setupSDKQueue(t, c, created.Arn) + + desc, err := c.DescribeSchedulingPolicies(ctx, &awsbatch.DescribeSchedulingPoliciesInput{Arns: []string{aws.ToString(created.Arn)}}) + if err != nil || len(desc.SchedulingPolicies) != 1 || + aws.ToInt32(desc.SchedulingPolicies[0].FairsharePolicy.ShareDecaySeconds) != 3600 || + desc.SchedulingPolicies[0].Tags["k"] != "v" { + t.Fatalf("DescribeSchedulingPolicies: %+v, %v", desc, err) + } + + if _, err := c.UpdateSchedulingPolicy(ctx, &awsbatch.UpdateSchedulingPolicyInput{ + Arn: created.Arn, FairsharePolicy: &batchtypes.FairsharePolicy{ShareDecaySeconds: aws.Int32(60)}, + }); err != nil { + t.Fatalf("UpdateSchedulingPolicy: %v", err) + } + + list, err := c.ListSchedulingPolicies(ctx, &awsbatch.ListSchedulingPoliciesInput{}) + if err != nil || len(list.SchedulingPolicies) != 1 || aws.ToString(list.SchedulingPolicies[0].Arn) != aws.ToString(created.Arn) { + t.Fatalf("ListSchedulingPolicies: %+v, %v", list, err) + } + + // Fair-share queue: shareIdentifier is required. + _, err = c.SubmitJob(ctx, &awsbatch.SubmitJobInput{JobName: aws.String("j"), JobQueue: aws.String("q"), JobDefinition: aws.String("jd")}) + requireClientException(t, err, "SubmitJob without shareIdentifier") + + if _, err := c.SubmitJob(ctx, &awsbatch.SubmitJobInput{ + JobName: aws.String("j"), JobQueue: aws.String("q"), JobDefinition: aws.String("jd"), ShareIdentifier: aws.String("A1"), + }); err != nil { + t.Fatalf("SubmitJob with shareIdentifier: %v", err) + } + + // In-use policy can't be deleted. + _, err = c.DeleteSchedulingPolicy(ctx, &awsbatch.DeleteSchedulingPolicyInput{Arn: created.Arn}) + requireClientException(t, err, "DeleteSchedulingPolicy in use") + + // A queue can't reference a missing policy. + _, err = c.CreateJobQueue(ctx, &awsbatch.CreateJobQueueInput{ + JobQueueName: aws.String("q2"), + Priority: aws.Int32(1), + SchedulingPolicyArn: aws.String("arn:aws:batch:us-east-1:000000000000:scheduling-policy/none"), + ComputeEnvironmentOrder: []batchtypes.ComputeEnvironmentOrder{ + {Order: aws.Int32(1), ComputeEnvironment: aws.String("ce")}, + }, + }) + requireClientException(t, err, "CreateJobQueue unknown policy") + + // Tags on the policy ARN. + tags, err := c.ListTagsForResource(ctx, &awsbatch.ListTagsForResourceInput{ResourceArn: created.Arn}) + if err != nil || tags.Tags["k"] != "v" { + t.Fatalf("ListTagsForResource(policy): %+v, %v", tags, err) + } +} + +func TestSDKRegisterJobDefinitionInvalidType(t *testing.T) { + ctx := context.Background() + c := newBatchClient(t) + + _, err := c.RegisterJobDefinition(ctx, &awsbatch.RegisterJobDefinitionInput{ + JobDefinitionName: aws.String("bad"), + Type: batchtypes.JobDefinitionType("batch"), + ContainerProperties: &batchtypes.ContainerProperties{Image: aws.String("busybox")}, + }) + requireClientException(t, err, "RegisterJobDefinition invalid type") + + _, err = c.RegisterJobDefinition(ctx, &awsbatch.RegisterJobDefinitionInput{ + JobDefinitionName: aws.String("mn"), + Type: batchtypes.JobDefinitionTypeMultinode, + }) + requireClientException(t, err, "RegisterJobDefinition multinode without nodeProperties") +} diff --git a/server/aws/batch/scheduling_policy.go b/server/aws/batch/scheduling_policy.go new file mode 100644 index 000000000..13a1fd7b1 --- /dev/null +++ b/server/aws/batch/scheduling_policy.go @@ -0,0 +1,104 @@ +package batch + +import ( + "net/http" + + "github.com/stackshy/cloudemu/v2/services/batch/driver" +) + +func (h *Handler) createSchedulingPolicy(w http.ResponseWriter, r *http.Request) { + var req createSchedulingPolicyRequest + if !decodeJSON(w, r, &req) { + return + } + + p, err := h.batch.CreateSchedulingPolicy(r.Context(), req.toInput()) + if err != nil { + writeErr(w, err) + + return + } + + writeJSON(w, schedulingPolicyNameArnResponse{Name: p.Name, Arn: p.ARN}) +} + +func (h *Handler) describeSchedulingPolicies(w http.ResponseWriter, r *http.Request) { + var req describeSchedulingPoliciesRequest + if !decodeJSON(w, r, &req) { + return + } + + policies, err := h.batch.DescribeSchedulingPolicies(r.Context(), req.Arns) + if err != nil { + writeErr(w, err) + + return + } + + details := make([]schedulingPolicyDetail, 0, len(policies)) + for i := range policies { + details = append(details, toSPDetail(&policies[i])) + } + + writeJSON(w, describeSchedulingPoliciesResponse{SchedulingPolicies: details}) +} + +func (h *Handler) listSchedulingPolicies(w http.ResponseWriter, r *http.Request) { + var req listSchedulingPoliciesRequest + if !decodeJSON(w, r, &req) { + return + } + + policies, err := h.batch.ListSchedulingPolicies(r.Context()) + if err != nil { + writeErr(w, err) + + return + } + + page, next, ok := paginate(w, policies, req.MaxResults, req.NextToken) + if !ok { + return + } + + out := make([]schedulingPolicyListing, 0, len(page)) + for i := range page { + out = append(out, schedulingPolicyListing{Arn: page[i].ARN}) + } + + writeJSON(w, listSchedulingPoliciesResponse{SchedulingPolicies: out, NextToken: next}) +} + +func (h *Handler) updateSchedulingPolicy(w http.ResponseWriter, r *http.Request) { + var req updateSchedulingPolicyRequest + if !decodeJSON(w, r, &req) { + return + } + + if err := h.batch.UpdateSchedulingPolicy(r.Context(), driver.UpdateSchedulingPolicyInput{ + ARN: req.Arn, + FairsharePolicy: req.FairsharePolicy, + QuotaSharePolicy: req.QuotaSharePolicy, + }); err != nil { + writeErr(w, err) + + return + } + + writeJSON(w, emptyResponse{}) +} + +func (h *Handler) deleteSchedulingPolicy(w http.ResponseWriter, r *http.Request) { + var req deleteSchedulingPolicyRequest + if !decodeJSON(w, r, &req) { + return + } + + if err := h.batch.DeleteSchedulingPolicy(r.Context(), req.Arn); err != nil { + writeErr(w, err) + + return + } + + writeJSON(w, emptyResponse{}) +} diff --git a/server/aws/batch/types_jobs.go b/server/aws/batch/types_jobs.go new file mode 100644 index 000000000..0e5ce93be --- /dev/null +++ b/server/aws/batch/types_jobs.go @@ -0,0 +1,326 @@ +package batch + +import ( + "encoding/json" + + "github.com/stackshy/cloudemu/v2/services/batch/driver" +) + +// --- Scheduling policies --- + +type createSchedulingPolicyRequest struct { + Name string `json:"name"` + FairsharePolicy json.RawMessage `json:"fairsharePolicy"` + QuotaSharePolicy json.RawMessage `json:"quotaSharePolicy"` + Tags map[string]string `json:"tags"` +} + +func (req *createSchedulingPolicyRequest) toInput() driver.CreateSchedulingPolicyInput { + return driver.CreateSchedulingPolicyInput{ + Name: req.Name, + FairsharePolicy: req.FairsharePolicy, + QuotaSharePolicy: req.QuotaSharePolicy, + Tags: req.Tags, + } +} + +type schedulingPolicyNameArnResponse struct { + Name string `json:"name"` + Arn string `json:"arn"` +} + +type describeSchedulingPoliciesRequest struct { + Arns []string `json:"arns"` +} + +type schedulingPolicyDetail struct { + Name string `json:"name"` + Arn string `json:"arn"` + FairsharePolicy json.RawMessage `json:"fairsharePolicy,omitempty"` + QuotaSharePolicy json.RawMessage `json:"quotaSharePolicy,omitempty"` + Tags map[string]string `json:"tags,omitempty"` +} + +type describeSchedulingPoliciesResponse struct { + SchedulingPolicies []schedulingPolicyDetail `json:"schedulingPolicies"` +} + +type listSchedulingPoliciesRequest struct { + MaxResults *int32 `json:"maxResults"` + NextToken string `json:"nextToken"` +} + +type schedulingPolicyListing struct { + Arn string `json:"arn"` +} + +type listSchedulingPoliciesResponse struct { + SchedulingPolicies []schedulingPolicyListing `json:"schedulingPolicies"` + NextToken string `json:"nextToken,omitempty"` +} + +type updateSchedulingPolicyRequest struct { + Arn string `json:"arn"` + FairsharePolicy json.RawMessage `json:"fairsharePolicy"` + QuotaSharePolicy json.RawMessage `json:"quotaSharePolicy"` +} + +type deleteSchedulingPolicyRequest struct { + Arn string `json:"arn"` +} + +func toSPDetail(p *driver.SchedulingPolicy) schedulingPolicyDetail { + return schedulingPolicyDetail{ + Name: p.Name, + Arn: p.ARN, + FairsharePolicy: p.FairsharePolicy, + QuotaSharePolicy: p.QuotaSharePolicy, + Tags: p.Tags, + } +} + +// --- Jobs --- + +type jobDependencyWire struct { + JobID string `json:"jobId,omitempty"` + Type string `json:"type,omitempty"` +} + +type arrayPropertiesRequest struct { + Size *int32 `json:"size"` +} + +type submitJobRequest struct { + JobName string `json:"jobName"` + JobQueue string `json:"jobQueue"` + JobDefinition string `json:"jobDefinition"` + ArrayProperties *arrayPropertiesRequest `json:"arrayProperties"` + DependsOn []jobDependencyWire `json:"dependsOn"` + Parameters map[string]string `json:"parameters"` + ContainerOverrides json.RawMessage `json:"containerOverrides"` + NodeOverrides json.RawMessage `json:"nodeOverrides"` + EcsPropertiesOverride json.RawMessage `json:"ecsPropertiesOverride"` + EksPropertiesOverride json.RawMessage `json:"eksPropertiesOverride"` + RetryStrategy json.RawMessage `json:"retryStrategy"` + Timeout json.RawMessage `json:"timeout"` + PropagateTags *bool `json:"propagateTags"` + SchedulingPriorityOverride *int32 `json:"schedulingPriorityOverride"` + ShareIdentifier string `json:"shareIdentifier"` + Tags map[string]string `json:"tags"` +} + +type submitJobResponse struct { + JobArn string `json:"jobArn"` + JobID string `json:"jobId"` + JobName string `json:"jobName"` +} + +type describeJobsRequest struct { + Jobs []string `json:"jobs"` +} + +type arrayPropertiesWire struct { + Size int32 `json:"size,omitempty"` + Index *int32 `json:"index,omitempty"` + StatusSummary map[string]int32 `json:"statusSummary,omitempty"` +} + +type jobDetail struct { + JobArn string `json:"jobArn"` + JobName string `json:"jobName"` + JobID string `json:"jobId"` + JobQueue string `json:"jobQueue"` + Status string `json:"status"` + StatusReason string `json:"statusReason,omitempty"` + Attempts []json.RawMessage `json:"attempts"` + CreatedAt int64 `json:"createdAt,omitempty"` + StartedAt int64 `json:"startedAt,omitempty"` + StoppedAt int64 `json:"stoppedAt,omitempty"` + DependsOn []jobDependencyWire `json:"dependsOn"` + JobDefinition string `json:"jobDefinition"` + Parameters map[string]string `json:"parameters"` + Container json.RawMessage `json:"container,omitempty"` + NodeProperties json.RawMessage `json:"nodeProperties,omitempty"` + EcsProperties json.RawMessage `json:"ecsProperties,omitempty"` + EksProperties json.RawMessage `json:"eksProperties,omitempty"` + ArrayProperties *arrayPropertiesWire `json:"arrayProperties,omitempty"` + RetryStrategy json.RawMessage `json:"retryStrategy,omitempty"` + Timeout json.RawMessage `json:"timeout,omitempty"` + PlatformCapabilities []string `json:"platformCapabilities"` + PropagateTags *bool `json:"propagateTags,omitempty"` + SchedulingPriority *int32 `json:"schedulingPriority,omitempty"` + ShareIdentifier string `json:"shareIdentifier,omitempty"` + IsCancelled *bool `json:"isCancelled,omitempty"` + IsTerminated *bool `json:"isTerminated,omitempty"` + Tags map[string]string `json:"tags,omitempty"` +} + +type describeJobsResponse struct { + Jobs []jobDetail `json:"jobs"` +} + +type jobFilterWire struct { + Name string `json:"name"` + Values []string `json:"values"` +} + +type listJobsRequest struct { + JobQueue string `json:"jobQueue"` + ArrayJobID string `json:"arrayJobId"` + MultiNodeJobID string `json:"multiNodeJobId"` + JobStatus string `json:"jobStatus"` + Filters []jobFilterWire `json:"filters"` + MaxResults *int32 `json:"maxResults"` + NextToken string `json:"nextToken"` +} + +type containerSummary struct { + ExitCode *int32 `json:"exitCode,omitempty"` +} + +type jobSummary struct { + JobArn string `json:"jobArn"` + JobID string `json:"jobId"` + JobName string `json:"jobName"` + CreatedAt int64 `json:"createdAt,omitempty"` + Status string `json:"status"` + StatusReason string `json:"statusReason,omitempty"` + StartedAt int64 `json:"startedAt,omitempty"` + StoppedAt int64 `json:"stoppedAt,omitempty"` + Container *containerSummary `json:"container,omitempty"` + ArrayProperties *arrayPropertiesWire `json:"arrayProperties,omitempty"` + JobDefinition string `json:"jobDefinition,omitempty"` + ShareIdentifier string `json:"shareIdentifier,omitempty"` + IsCancelled *bool `json:"isCancelled,omitempty"` + IsTerminated *bool `json:"isTerminated,omitempty"` +} + +type listJobsResponse struct { + JobSummaryList []jobSummary `json:"jobSummaryList"` + NextToken string `json:"nextToken,omitempty"` +} + +type stopJobRequest struct { + JobID string `json:"jobId"` + Reason string `json:"reason"` +} + +func depsToDriver(in []jobDependencyWire) []driver.JobDependency { + if in == nil { + return nil + } + + out := make([]driver.JobDependency, len(in)) + for i, d := range in { + out[i] = driver.JobDependency{JobID: d.JobID, Type: d.Type} + } + + return out +} + +func depsToWire(in []driver.JobDependency) []jobDependencyWire { + out := make([]jobDependencyWire, len(in)) + for i, d := range in { + out[i] = jobDependencyWire{JobID: d.JobID, Type: d.Type} + } + + return out +} + +func filtersToDriver(in []jobFilterWire) []driver.JobFilter { + if in == nil { + return nil + } + + out := make([]driver.JobFilter, len(in)) + for i, f := range in { + out[i] = driver.JobFilter{Name: f.Name, Values: f.Values} + } + + return out +} + +func arrayToWire(ap *driver.ArrayProperties) *arrayPropertiesWire { + if ap == nil { + return nil + } + + return &arrayPropertiesWire{Size: ap.Size, Index: ap.Index, StatusSummary: ap.StatusSummary} +} + +func boolPtrIf(b bool) *bool { + if !b { + return nil + } + + return &b +} + +func toJobDetail(j *driver.Job) jobDetail { + params := j.Parameters + if params == nil { + params = map[string]string{} + } + + caps := j.PlatformCapabilities + if caps == nil { + caps = []string{} + } + + return jobDetail{ + JobArn: j.JobARN, + JobName: j.JobName, + JobID: j.JobID, + JobQueue: j.JobQueue, + Status: j.Status, + StatusReason: j.StatusReason, + Attempts: []json.RawMessage{}, + CreatedAt: j.CreatedAt, + StartedAt: j.StartedAt, + StoppedAt: j.StoppedAt, + DependsOn: depsToWire(j.DependsOn), + JobDefinition: j.JobDefinition, + Parameters: params, + Container: j.Container, + NodeProperties: j.NodeProperties, + EcsProperties: j.EcsProperties, + EksProperties: j.EksProperties, + ArrayProperties: arrayToWire(j.ArrayProperties), + RetryStrategy: j.RetryStrategy, + Timeout: j.Timeout, + PlatformCapabilities: caps, + PropagateTags: j.PropagateTags, + SchedulingPriority: j.SchedulingPriority, + ShareIdentifier: j.ShareIdentifier, + IsCancelled: boolPtrIf(j.IsCancelled), + IsTerminated: boolPtrIf(j.IsTerminated), + Tags: j.Tags, + } +} + +// toJobSummary renders the ListJobs JobSummary of a job. container.exitCode is +// reported once the job's container has exited successfully. +func toJobSummary(j *driver.Job) jobSummary { + s := jobSummary{ + JobArn: j.JobARN, + JobID: j.JobID, + JobName: j.JobName, + CreatedAt: j.CreatedAt, + Status: j.Status, + StatusReason: j.StatusReason, + StartedAt: j.StartedAt, + StoppedAt: j.StoppedAt, + ArrayProperties: arrayToWire(j.ArrayProperties), + JobDefinition: j.JobDefinition, + ShareIdentifier: j.ShareIdentifier, + IsCancelled: boolPtrIf(j.IsCancelled), + IsTerminated: boolPtrIf(j.IsTerminated), + } + + if j.Status == driver.JobStatusSucceeded && j.Container != nil && (j.ArrayProperties == nil || j.ArrayProperties.Index != nil) { + zero := int32(0) + s.Container = &containerSummary{ExitCode: &zero} + } + + return s +} diff --git a/services/batch/driver/driver.go b/services/batch/driver/driver.go index 9d4e91df9..445de672f 100644 --- a/services/batch/driver/driver.go +++ b/services/batch/driver/driver.go @@ -1,6 +1,7 @@ // Package driver defines the interface and types for AWS Batch implementations. -// It models the Batch control plane: compute environments, job queues, and -// job-definition revisions, plus resource tagging. +// It models the Batch control plane: compute environments, job queues, +// job-definition revisions, fair-share scheduling policies, and the job +// lifecycle (submit/describe/list/cancel/terminate), plus resource tagging. // // The interface is deliberately narrow: a backend stores what it is given and // echoes it back, generating ARNs, ECS cluster ARNs, and job-definition @@ -31,7 +32,21 @@ type Batch interface { DescribeJobDefinitions(ctx context.Context, in DescribeJobDefinitionsInput) ([]JobDefinition, error) DeregisterJobDefinition(ctx context.Context, nameRevisionOrARN string) error - // Tags (by ARN), shared across all three resource kinds. + // Scheduling policies. + CreateSchedulingPolicy(ctx context.Context, in CreateSchedulingPolicyInput) (*SchedulingPolicy, error) + DescribeSchedulingPolicies(ctx context.Context, arns []string) ([]SchedulingPolicy, error) + ListSchedulingPolicies(ctx context.Context) ([]SchedulingPolicy, error) + UpdateSchedulingPolicy(ctx context.Context, in UpdateSchedulingPolicyInput) error + DeleteSchedulingPolicy(ctx context.Context, arn string) error + + // Jobs. + SubmitJob(ctx context.Context, in SubmitJobInput) (*Job, error) + DescribeJobs(ctx context.Context, jobIDs []string) ([]Job, error) + ListJobs(ctx context.Context, in ListJobsInput) (*ListJobsOutput, error) + CancelJob(ctx context.Context, jobID, reason string) error + TerminateJob(ctx context.Context, jobID, reason string) error + + // Tags (by ARN), shared across every taggable resource kind. TagResource(ctx context.Context, arn string, tags map[string]string) error UntagResource(ctx context.Context, arn string, keys []string) error ListTagsForResource(ctx context.Context, arn string) (map[string]string, error) diff --git a/services/batch/driver/types.go b/services/batch/driver/types.go index ae88c9136..ea5d6000b 100644 --- a/services/batch/driver/types.go +++ b/services/batch/driver/types.go @@ -173,3 +173,155 @@ type DescribeJobDefinitionsInput struct { Name string Status string } + +// SchedulingPolicy is the server-side detail of an AWS Batch scheduling policy +// (DescribeSchedulingPolicies SchedulingPolicyDetail shape). The fair-share and +// quota-share policy blobs are carried verbatim as raw JSON so every nested +// field round-trips exactly. +type SchedulingPolicy struct { + Name string + ARN string + FairsharePolicy json.RawMessage + QuotaSharePolicy json.RawMessage + Tags map[string]string +} + +// CreateSchedulingPolicyInput describes a scheduling policy to create. +type CreateSchedulingPolicyInput struct { + Name string + FairsharePolicy json.RawMessage + QuotaSharePolicy json.RawMessage + Tags map[string]string +} + +// UpdateSchedulingPolicyInput replaces a scheduling policy's fair-share or +// quota-share details. A nil blob leaves that policy unchanged. +type UpdateSchedulingPolicyInput struct { + ARN string + FairsharePolicy json.RawMessage + QuotaSharePolicy json.RawMessage +} + +// Job statuses (the JobStatus enum). A job moves SUBMITTED -> (PENDING while +// it waits on dependencies) -> RUNNABLE -> STARTING -> RUNNING and ends in +// SUCCEEDED or FAILED. +const ( + JobStatusSubmitted = "SUBMITTED" + JobStatusPending = "PENDING" + JobStatusRunnable = "RUNNABLE" + JobStatusStarting = "STARTING" + JobStatusRunning = "RUNNING" + JobStatusSucceeded = "SUCCEEDED" + JobStatusFailed = "FAILED" +) + +// Array-job dependency types (the ArrayJobDependency enum). +const ( + DependencyNToN = "N_TO_N" + DependencySequential = "SEQUENTIAL" +) + +// ListJobs filter names (the KeyValuesPair names ListJobs accepts). +const ( + FilterJobName = "JOB_NAME" + FilterJobDefinition = "JOB_DEFINITION" + FilterBeforeCreatedAt = "BEFORE_CREATED_AT" + FilterAfterCreatedAt = "AFTER_CREATED_AT" + FilterShareIdentifier = "SHARE_IDENTIFIER" +) + +// JobDependency is one entry of a job's dependsOn list. +type JobDependency struct { + JobID string + Type string +} + +// ArrayProperties is the array-job section of a job. A parent carries Size and +// StatusSummary (child count per status); a child carries Index. +type ArrayProperties struct { + Size int32 + Index *int32 + StatusSummary map[string]int32 +} + +// SubmitJobInput describes a job to submit. JobQueue and JobDefinition accept +// a name or ARN (a job definition with or without a revision). A nil ArraySize +// submits a single (non-array) job. +type SubmitJobInput struct { + JobName string + JobQueue string + JobDefinition string + ArraySize *int32 + DependsOn []JobDependency + Parameters map[string]string + ContainerOverrides json.RawMessage + NodeOverrides json.RawMessage + EcsPropertiesOverride json.RawMessage + EksPropertiesOverride json.RawMessage + RetryStrategy json.RawMessage + Timeout json.RawMessage + PropagateTags *bool + SchedulingPriorityOverride *int32 + ShareIdentifier string + Tags map[string]string +} + +// Job is the server-side detail of an AWS Batch job as DescribeJobs reports it +// (JobDetail shape) at the instant it is observed. Times are epoch +// milliseconds; zero means unset. Container, NodeProperties, EcsProperties and +// EksProperties are the job definition's blobs (with container overrides +// applied) carried as raw JSON. +type Job struct { + JobID string + JobARN string + JobName string + JobQueue string + JobDefinition string + Status string + StatusReason string + CreatedAt int64 + StartedAt int64 + StoppedAt int64 + DependsOn []JobDependency + ArrayProperties *ArrayProperties + Parameters map[string]string + Container json.RawMessage + NodeProperties json.RawMessage + EcsProperties json.RawMessage + EksProperties json.RawMessage + RetryStrategy json.RawMessage + Timeout json.RawMessage + PlatformCapabilities []string + PropagateTags *bool + SchedulingPriority *int32 + ShareIdentifier string + IsCancelled bool + IsTerminated bool + Tags map[string]string +} + +// JobFilter is one ListJobs filter (KeyValuesPair). +type JobFilter struct { + Name string + Values []string +} + +// ListJobsInput selects the jobs ListJobs returns. Exactly one of JobQueue, +// ArrayJobID or MultiNodeJobID must be set. JobStatus defaults to RUNNING and +// is ignored when a filter (other than SHARE_IDENTIFIER) is given. A nil +// MaxResults pages at the service default; NextToken continues a listing. +type ListJobsInput struct { + JobQueue string + ArrayJobID string + MultiNodeJobID string + JobStatus string + Filters []JobFilter + MaxResults *int32 + NextToken string +} + +// ListJobsOutput is one page of ListJobs. NextToken is empty on the last page. +type ListJobsOutput struct { + Jobs []Job + NextToken string +} From 063a04d107eab3f975708255a26ee2e3f231a7f0 Mon Sep 17 00:00:00 2001 From: Satyam Trivedi Date: Fri, 16 Oct 2026 00:09:05 +0530 Subject: [PATCH 2/4] docs(coverage): regenerate for the Batch job and scheduling-policy operations Part of #1496 --- docs/coverage/README.md | 2 +- docs/coverage/aws/README.md | 2 +- docs/coverage/aws/batch.md | 21 +++++++++++++++++--- docs/coverage/coverage.json | 34 ++++++++++++++++++++++++++++++++- docs/coverage/nongoals/batch.md | 6 ++++++ 5 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 docs/coverage/nongoals/batch.md diff --git a/docs/coverage/README.md b/docs/coverage/README.md index 1f226d00d..3a6db0aa8 100644 --- a/docs/coverage/README.md +++ b/docs/coverage/README.md @@ -33,7 +33,7 @@ code does not implement. Machine-readable: [`coverage.json`](./coverage.json). | `backup` | [Backup](./aws/backup.md) | - | - | - | 25 | | `backupdr` | - | - | [BackupDR](./gcp/backupdr.md) | - | 6 | | `bastion` | - | [Bastion](./azure/bastion.md) | - | - | 4 | -| `batch` | [Batch](./aws/batch.md) | - | - | - | 14 | +| `batch` | [Batch](./aws/batch.md) | - | - | - | 24 | | `batch-azure` | - | [Batch](./azure/batch.md) | - | - | 17 | | `bedrock` | [Bedrock](./aws/bedrock.md) | - | - | - | 65 | | `bedrockagent` | [BedrockAgent](./aws/bedrockagent.md) | - | - | - | 32 | diff --git a/docs/coverage/aws/README.md b/docs/coverage/aws/README.md index daa69bc47..9583d0ad5 100644 --- a/docs/coverage/aws/README.md +++ b/docs/coverage/aws/README.md @@ -15,7 +15,7 @@ Services cloudemu emulates for AWS, by native name. Back to the [cross-provider | [AppSync](./appsync.md) | `appsync` | 17 | | [Athena](./athena.md) | `athena` | 26 | | [Backup](./backup.md) | `backup` | 25 | -| [Batch](./batch.md) | `batch` | 14 | +| [Batch](./batch.md) | `batch` | 24 | | [Bedrock](./bedrock.md) | `bedrock` | 65 | | [BedrockAgent](./bedrockagent.md) | `bedrockagent` | 32 | | [BedrockAgentRuntime](./bedrockagentruntime.md) | `bedrockagentruntime` | 3 | diff --git a/docs/coverage/aws/batch.md b/docs/coverage/aws/batch.md index 20101f174..c8db1fb55 100644 --- a/docs/coverage/aws/batch.md +++ b/docs/coverage/aws/batch.md @@ -3,25 +3,40 @@ AWS's `batch` service · portable interface `driver.Batch` · [AWS index](./README.md) -## Operations (14) +## Operations (24) | Operation | Description | | --- | --- | +| `CancelJob` | | | `CreateComputeEnvironment` | Compute environments. | | `CreateJobQueue` | Job queues. | +| `CreateSchedulingPolicy` | Scheduling policies. | | `DeleteComputeEnvironment` | | | `DeleteJobQueue` | | +| `DeleteSchedulingPolicy` | | | `DeregisterJobDefinition` | | | `DescribeComputeEnvironments` | | | `DescribeJobDefinitions` | | | `DescribeJobQueues` | | +| `DescribeJobs` | | +| `DescribeSchedulingPolicies` | | +| `ListJobs` | | +| `ListSchedulingPolicies` | | | `ListTagsForResource` | | | `RegisterJobDefinition` | Job definitions. | -| `TagResource` | Tags (by ARN), shared across all three resource kinds. | +| `SubmitJob` | Jobs. | +| `TagResource` | Tags (by ARN), shared across every taggable resource kind. | +| `TerminateJob` | | | `UntagResource` | | | `UpdateComputeEnvironment` | | | `UpdateJobQueue` | | +| `UpdateSchedulingPolicy` | | ## Not in scope -_Not documented yet. See the [emulator boundary](../../../README.md) for cloudemu-wide non-goals._ +- Jobs run no workload: a job always succeeds (container `exitCode` 0) once its dependencies succeed. By default it is `SUCCEEDED` as soon as it is submitted; with `--async-settle` it spends about one second each in `SUBMITTED`, `RUNNABLE`, `STARTING` and `RUNNING` (and stays `PENDING` while a dependency is unfinished). `attempts` is always empty, and retry strategies and timeouts are stored and echoed but never applied. +- `CancelJob` on a `PENDING` job takes effect at once. Real Batch waits for the job's dependencies to finish first. +- Array jobs: children are derived from the parent, not stored one by one. `SEQUENTIAL` and `N_TO_N` dependencies are honoured. An array parent reports `PENDING` while any child is unfinished. Multi-node parallel jobs run as a single job: `ListJobs` with `multiNodeJobId` returns an empty list. +- Fair-share scheduling policies are stored, validated and bound to queues, but they never change the order in which jobs run. Jobs never stay `RUNNABLE` because no capacity is free, so "Batch Job Queue Blocked" events are never sent. +- "Batch Job State Change" events go to the EventBridge default bus on the serve scheduler's tick (and on any Batch job API call), one per status the job passed through; none is sent for the initial submission. In library mode, nothing ticks, so events are published when a job API call sees the change or when `Tick` is called. A job that fails reports only `FAILED`, without the earlier steps. Array children send no events of their own. Real Batch sends no state-change events for compute environments or job queues, so the emulator sends none either. +- Error messages are written by the emulator. The exception type (`ClientException`, HTTP 400) is the documented one. diff --git a/docs/coverage/coverage.json b/docs/coverage/coverage.json index f8c4de6e9..6e1ad5147 100644 --- a/docs/coverage/coverage.json +++ b/docs/coverage/coverage.json @@ -2655,6 +2655,9 @@ "service": "batch", "interface": "Batch", "operations": [ + { + "name": "CancelJob" + }, { "name": "CreateComputeEnvironment", "doc": "Compute environments." @@ -2663,12 +2666,19 @@ "name": "CreateJobQueue", "doc": "Job queues." }, + { + "name": "CreateSchedulingPolicy", + "doc": "Scheduling policies." + }, { "name": "DeleteComputeEnvironment" }, { "name": "DeleteJobQueue" }, + { + "name": "DeleteSchedulingPolicy" + }, { "name": "DeregisterJobDefinition" }, @@ -2681,6 +2691,18 @@ { "name": "DescribeJobQueues" }, + { + "name": "DescribeJobs" + }, + { + "name": "DescribeSchedulingPolicies" + }, + { + "name": "ListJobs" + }, + { + "name": "ListSchedulingPolicies" + }, { "name": "ListTagsForResource" }, @@ -2688,9 +2710,16 @@ "name": "RegisterJobDefinition", "doc": "Job definitions." }, + { + "name": "SubmitJob", + "doc": "Jobs." + }, { "name": "TagResource", - "doc": "Tags (by ARN), shared across all three resource kinds." + "doc": "Tags (by ARN), shared across every taggable resource kind." + }, + { + "name": "TerminateJob" }, { "name": "UntagResource" @@ -2700,6 +2729,9 @@ }, { "name": "UpdateJobQueue" + }, + { + "name": "UpdateSchedulingPolicy" } ], "providers": { diff --git a/docs/coverage/nongoals/batch.md b/docs/coverage/nongoals/batch.md new file mode 100644 index 000000000..2eb8ca880 --- /dev/null +++ b/docs/coverage/nongoals/batch.md @@ -0,0 +1,6 @@ +- Jobs run no workload: a job always succeeds (container `exitCode` 0) once its dependencies succeed. By default it is `SUCCEEDED` as soon as it is submitted; with `--async-settle` it spends about one second each in `SUBMITTED`, `RUNNABLE`, `STARTING` and `RUNNING` (and stays `PENDING` while a dependency is unfinished). `attempts` is always empty, and retry strategies and timeouts are stored and echoed but never applied. +- `CancelJob` on a `PENDING` job takes effect at once. Real Batch waits for the job's dependencies to finish first. +- Array jobs: children are derived from the parent, not stored one by one. `SEQUENTIAL` and `N_TO_N` dependencies are honoured. An array parent reports `PENDING` while any child is unfinished. Multi-node parallel jobs run as a single job: `ListJobs` with `multiNodeJobId` returns an empty list. +- Fair-share scheduling policies are stored, validated and bound to queues, but they never change the order in which jobs run. Jobs never stay `RUNNABLE` because no capacity is free, so "Batch Job Queue Blocked" events are never sent. +- "Batch Job State Change" events go to the EventBridge default bus on the serve scheduler's tick (and on any Batch job API call), one per status the job passed through; none is sent for the initial submission. In library mode, nothing ticks, so events are published when a job API call sees the change or when `Tick` is called. A job that fails reports only `FAILED`, without the earlier steps. Array children send no events of their own. Real Batch sends no state-change events for compute environments or job queues, so the emulator sends none either. +- Error messages are written by the emulator. The exception type (`ClientException`, HTTP 400) is the documented one. From 1628072fe818a975943bb2443b945e9ff1583582 Mon Sep 17 00:00:00 2001 From: Satyam Trivedi Date: Fri, 16 Oct 2026 00:25:48 +0530 Subject: [PATCH 3/4] fix(aws-batch): bound array child index to int32 when parsing job IDs Parse the array child index with strconv.ParseInt(..., 10, 32) so the value converted into ArrayProperties.Index is bounded at parse time (CodeQL go/incorrect-integer-conversion). lookup already rejected any index at or above the array size, so no out-of-range value reached the conversion; this makes the bound explicit. --- providers/aws/batch/job_lifecycle.go | 6 ++++-- providers/aws/batch/jobs_test.go | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/providers/aws/batch/job_lifecycle.go b/providers/aws/batch/job_lifecycle.go index dd2536897..5cf2b95b2 100644 --- a/providers/aws/batch/job_lifecycle.go +++ b/providers/aws/batch/job_lifecycle.go @@ -184,12 +184,14 @@ func splitChildID(id string) (parent string, idx int, ok bool) { return "", 0, false } - n, err := strconv.Atoi(id[i+1:]) + // Parse with a 32-bit bound: the index ends up in the int32 + // ArrayProperties.Index field. + n, err := strconv.ParseInt(id[i+1:], 10, 32) if err != nil || n < 0 { return "", 0, false } - return id[:i], n, true + return id[:i], int(n), true } // lookup resolves a job ID to its record and, for an array child, its index diff --git a/providers/aws/batch/jobs_test.go b/providers/aws/batch/jobs_test.go index f22d53511..04b44a33b 100644 --- a/providers/aws/batch/jobs_test.go +++ b/providers/aws/batch/jobs_test.go @@ -395,6 +395,13 @@ func TestArrayJobChildren(t *testing.T) { t.Fatalf("child index missing: %+v", child) } + // An index past int32 must not wrap around to a real child. + for _, bad := range []string{":4294967296", ":2147483648", ":-1"} { + if jobs, err := m.DescribeJobs(ctx, []string{parent.JobID + bad}); err != nil || len(jobs) != 0 { + t.Fatalf("DescribeJobs(%s) = %+v, %v; want no job", bad, jobs, err) + } + } + all := driver.JobStatusPending kids, err := listJobs(m, driver.ListJobsInput{ArrayJobID: parent.JobID, JobStatus: all}) if err != nil || len(kids) != 2 { From db8553e7730fad22c7cdb89e4ead633834e601de Mon Sep 17 00:00:00 2001 From: Satyam Trivedi Date: Thu, 22 Oct 2026 12:19:15 +0530 Subject: [PATCH 4/4] fix(aws-batch): apply property overrides and address review findings - SubmitJob merges ecsPropertiesOverride, eksPropertiesOverride and nodeOverrides into the job detail instead of dropping them, and rejects an override the job definition can't take (wrong properties kind, unknown container name, numNodes without an open-ended range, targetNodes covering no range). - SubmitJob refuses an explicit INACTIVE revision (name:revision or ARN). - Replayed intermediate "Batch Job State Change" events no longer carry the final container exitCode. - With async settle a job stays RUNNING for 5s (DefaultBatchJobRunDwell) so a real client can terminate a running job; the dwell is stored on the job record and survives snapshots. - ListJobs with arrayJobId applies the JOB_NAME and other filters to the children. - Scheduling-policy binding (CreateJobQueue, UpdateJobQueue) and DeleteSchedulingPolicy are serialized so a queue can't end up bound to a deleted policy. - RegisterJobDefinition stores platformCapabilities as sent instead of echoing a default EC2, which made Terraform multinode definitions drift. --- docs/coverage/aws/batch.md | 3 +- docs/coverage/nongoals/batch.md | 3 +- internal/settle/settle.go | 11 +- providers/aws/batch/batch.go | 5 + providers/aws/batch/batch_test.go | 6 +- providers/aws/batch/events.go | 4 +- providers/aws/batch/job_definition.go | 12 +- providers/aws/batch/job_lifecycle.go | 57 +- providers/aws/batch/job_queue.go | 8 + providers/aws/batch/jobs.go | 26 +- providers/aws/batch/jobs_overrides_test.go | 367 +++++++++++++ providers/aws/batch/jobs_test.go | 55 +- providers/aws/batch/overrides.go | 609 +++++++++++++++++++++ providers/aws/batch/scheduling_policy.go | 3 + server/aws/batch/jobs_sdk_test.go | 51 ++ 15 files changed, 1171 insertions(+), 49 deletions(-) create mode 100644 providers/aws/batch/jobs_overrides_test.go create mode 100644 providers/aws/batch/overrides.go diff --git a/docs/coverage/aws/batch.md b/docs/coverage/aws/batch.md index c8db1fb55..2dbae14ca 100644 --- a/docs/coverage/aws/batch.md +++ b/docs/coverage/aws/batch.md @@ -34,9 +34,10 @@ AWS's `batch` service · portable interface `driver.Batch` · [AWS index](./READ ## Not in scope -- Jobs run no workload: a job always succeeds (container `exitCode` 0) once its dependencies succeed. By default it is `SUCCEEDED` as soon as it is submitted; with `--async-settle` it spends about one second each in `SUBMITTED`, `RUNNABLE`, `STARTING` and `RUNNING` (and stays `PENDING` while a dependency is unfinished). `attempts` is always empty, and retry strategies and timeouts are stored and echoed but never applied. +- Jobs run no workload: a job always succeeds (container `exitCode` 0) once its dependencies succeed. By default it is `SUCCEEDED` as soon as it is submitted; with `--async-settle` it spends about one second each in `SUBMITTED`, `RUNNABLE` and `STARTING`, then five seconds in `RUNNING` (and stays `PENDING` while a dependency is unfinished). `attempts` is always empty, and retry strategies and timeouts are stored and echoed but never applied. - `CancelJob` on a `PENDING` job takes effect at once. Real Batch waits for the job's dependencies to finish first. - Array jobs: children are derived from the parent, not stored one by one. `SEQUENTIAL` and `N_TO_N` dependencies are honoured. An array parent reports `PENDING` while any child is unfinished. Multi-node parallel jobs run as a single job: `ListJobs` with `multiNodeJobId` returns an empty list. - Fair-share scheduling policies are stored, validated and bound to queues, but they never change the order in which jobs run. Jobs never stay `RUNNABLE` because no capacity is free, so "Batch Job Queue Blocked" events are never sent. - "Batch Job State Change" events go to the EventBridge default bus on the serve scheduler's tick (and on any Batch job API call), one per status the job passed through; none is sent for the initial submission. In library mode, nothing ticks, so events are published when a job API call sees the change or when `Tick` is called. A job that fails reports only `FAILED`, without the earlier steps. Array children send no events of their own. Real Batch sends no state-change events for compute environments or job queues, so the emulator sends none either. - Error messages are written by the emulator. The exception type (`ClientException`, HTTP 400) is the documented one. +- `containerOverrides`, `ecsPropertiesOverride`, `eksPropertiesOverride` and `nodeOverrides` are merged into the job detail `DescribeJobs` returns (environment variables by name, resource requirements by type, other fields replaced). Overrides are not checked against compute-environment platform rules (for example `nodeOverrides` on Fargate), and multi-node jobs report no per-node `nodeDetails`. diff --git a/docs/coverage/nongoals/batch.md b/docs/coverage/nongoals/batch.md index 2eb8ca880..1cee8c5cd 100644 --- a/docs/coverage/nongoals/batch.md +++ b/docs/coverage/nongoals/batch.md @@ -1,6 +1,7 @@ -- Jobs run no workload: a job always succeeds (container `exitCode` 0) once its dependencies succeed. By default it is `SUCCEEDED` as soon as it is submitted; with `--async-settle` it spends about one second each in `SUBMITTED`, `RUNNABLE`, `STARTING` and `RUNNING` (and stays `PENDING` while a dependency is unfinished). `attempts` is always empty, and retry strategies and timeouts are stored and echoed but never applied. +- Jobs run no workload: a job always succeeds (container `exitCode` 0) once its dependencies succeed. By default it is `SUCCEEDED` as soon as it is submitted; with `--async-settle` it spends about one second each in `SUBMITTED`, `RUNNABLE` and `STARTING`, then five seconds in `RUNNING` (and stays `PENDING` while a dependency is unfinished). `attempts` is always empty, and retry strategies and timeouts are stored and echoed but never applied. - `CancelJob` on a `PENDING` job takes effect at once. Real Batch waits for the job's dependencies to finish first. - Array jobs: children are derived from the parent, not stored one by one. `SEQUENTIAL` and `N_TO_N` dependencies are honoured. An array parent reports `PENDING` while any child is unfinished. Multi-node parallel jobs run as a single job: `ListJobs` with `multiNodeJobId` returns an empty list. - Fair-share scheduling policies are stored, validated and bound to queues, but they never change the order in which jobs run. Jobs never stay `RUNNABLE` because no capacity is free, so "Batch Job Queue Blocked" events are never sent. - "Batch Job State Change" events go to the EventBridge default bus on the serve scheduler's tick (and on any Batch job API call), one per status the job passed through; none is sent for the initial submission. In library mode, nothing ticks, so events are published when a job API call sees the change or when `Tick` is called. A job that fails reports only `FAILED`, without the earlier steps. Array children send no events of their own. Real Batch sends no state-change events for compute environments or job queues, so the emulator sends none either. - Error messages are written by the emulator. The exception type (`ClientException`, HTTP 400) is the documented one. +- `containerOverrides`, `ecsPropertiesOverride`, `eksPropertiesOverride` and `nodeOverrides` are merged into the job detail `DescribeJobs` returns (environment variables by name, resource requirements by type, other fields replaced). Overrides are not checked against compute-environment platform rules (for example `nodeOverrides` on Fargate), and multi-node jobs report no per-node `nodeDetails`. diff --git a/internal/settle/settle.go b/internal/settle/settle.go index 3a7ba54df..0d122d90a 100644 --- a/internal/settle/settle.go +++ b/internal/settle/settle.go @@ -84,10 +84,15 @@ const ( DefaultRestoreStandardSettle = 3 * time.Second DefaultRestoreBulkSettle = 5 * time.Second - // DefaultBatchJobPhase is how long an AWS Batch job dwells in each - // non-terminal phase (SUBMITTED, RUNNABLE, STARTING, RUNNING) before moving - // on, so a dependency-free job reaches SUCCEEDED 4 phases after submission. + // DefaultBatchJobPhase is how long an AWS Batch job dwells in SUBMITTED, + // RUNNABLE and STARTING before moving on; RUNNING lasts + // DefaultBatchJobRunDwell. DefaultBatchJobPhase = 1 * time.Second + + // DefaultBatchJobRunDwell is how long an AWS Batch job stays RUNNING. It is + // longer than the other phases so a real CLI or SDK client can observe a + // running job and call TerminateJob on it before it finishes. + DefaultBatchJobRunDwell = 5 * time.Second ) // Window is a read-time overlay describing a resource still settling into its diff --git a/providers/aws/batch/batch.go b/providers/aws/batch/batch.go index d859144cf..6e6b761d7 100644 --- a/providers/aws/batch/batch.go +++ b/providers/aws/batch/batch.go @@ -51,6 +51,11 @@ type Mock struct { // RegisterJobDefinition calls for the same name can't claim the same revision. registerMu sync.Mutex + // policyBindMu serializes binding a scheduling policy to a job queue + // (CreateJobQueue, UpdateJobQueue) against DeleteSchedulingPolicy, so a + // policy can't be deleted between the in-use check and a queue binding it. + policyBindMu sync.Mutex + // jobsMu serializes job submission (sequence allocation), cancel/terminate // decisions and the event-emission bookkeeping on job records. jobsMu sync.Mutex diff --git a/providers/aws/batch/batch_test.go b/providers/aws/batch/batch_test.go index 6be9c0ee6..174dd5639 100644 --- a/providers/aws/batch/batch_test.go +++ b/providers/aws/batch/batch_test.go @@ -189,8 +189,10 @@ func TestRegisterJobDefinitionRevisionIncrements(t *testing.T) { t.Fatalf("first revision/status wrong: %+v", first) } - if len(first.PlatformCapabilities) != 1 || first.PlatformCapabilities[0] != "EC2" { - t.Fatalf("platformCapabilities default should be [EC2], got %v", first.PlatformCapabilities) + // No default is echoed when the caller sent none (it would drift in + // Terraform, which reads the list back as-is). + if len(first.PlatformCapabilities) != 0 { + t.Fatalf("platformCapabilities should be empty when none was sent, got %v", first.PlatformCapabilities) } second, err := m.RegisterJobDefinition(ctx, in) diff --git a/providers/aws/batch/events.go b/providers/aws/batch/events.go index 35d2ab091..f3947cf03 100644 --- a/providers/aws/batch/events.go +++ b/providers/aws/batch/events.go @@ -217,7 +217,9 @@ func eventDetail(j *driver.Job, status string) jobStateChangeDetail { if status != j.Status { // An intermediate status replayed after the fact: report only what was - // true then. + // true then. The container hadn't exited yet, so it has no exitCode. + d.Container = withoutExitCode(j.Container) + return d } diff --git a/providers/aws/batch/job_definition.go b/providers/aws/batch/job_definition.go index 6c034bd38..1c8bc54d7 100644 --- a/providers/aws/batch/job_definition.go +++ b/providers/aws/batch/job_definition.go @@ -28,7 +28,10 @@ func cloneJobDefinition(jd driver.JobDefinition) driver.JobDefinition { // RegisterJobDefinition registers a new revision. A repeated name yields the // next revision (N+1) with a new ARN; prior revisions remain ACTIVE until -// deregistered. platformCapabilities defaults to [EC2]. +// deregistered. platformCapabilities is stored as sent: real Batch treats an +// empty list as EC2 but doesn't echo a default, so DescribeJobDefinitions +// returns none (Terraform's acceptance tests assert platform_capabilities.# = 0 +// for container and multinode definitions registered without it). // //nolint:gocritic // in is the driver RegisterJobDefinitionInput, taken by value to match the interface func (m *Mock) RegisterJobDefinition( @@ -46,11 +49,6 @@ func (m *Mock) RegisterJobDefinition( return nil, err } - caps := in.PlatformCapabilities - if len(caps) == 0 { - caps = []string{"EC2"} - } - m.registerMu.Lock() cur, _ := m.jobDefMaxRev.Get(in.Name) @@ -69,7 +67,7 @@ func (m *Mock) RegisterJobDefinition( RetryStrategy: copyRaw(in.RetryStrategy), Timeout: copyRaw(in.Timeout), Parameters: copyTags(in.Parameters), - PlatformCapabilities: copyStrings(caps), + PlatformCapabilities: copyStrings(in.PlatformCapabilities), PropagateTags: copyBool(in.PropagateTags), SchedulingPriority: copyInt32(in.SchedulingPriority), Tags: copyTags(in.Tags), diff --git a/providers/aws/batch/job_lifecycle.go b/providers/aws/batch/job_lifecycle.go index 5cf2b95b2..f2c319853 100644 --- a/providers/aws/batch/job_lifecycle.go +++ b/providers/aws/batch/job_lifecycle.go @@ -19,11 +19,10 @@ const ( // Phase offsets, in units of a job's phase duration, measured from the instant // it became ready (dependencies met): RUNNABLE for one phase, STARTING for the -// next, RUNNING for the third, then SUCCEEDED. +// next, then RUNNING for the job's run dwell before SUCCEEDED. const ( - phasesToStarting = 1 - phasesToRunning = 2 - phasesToSucceeded = 3 + phasesToStarting = 1 + phasesToRunning = 2 ) // jobStop records a CancelJob/TerminateJob that took effect on a job (or one @@ -42,8 +41,10 @@ type jobStop struct { // settle overlay derives other services' transient states: no goroutines or // timers, advancing the clock is what moves a job through its lifecycle. // -// Phase is the dwell time in each non-terminal phase; zero (AsyncSettle off) -// means the job is SUCCEEDED as soon as its dependencies are. Array children +// Phase is the dwell time in each non-terminal phase except RUNNING, which +// lasts RunPhase; zero (AsyncSettle off) means the job is SUCCEEDED as soon as +// its dependencies are. A record restored from a snapshot taken before +// RunPhase existed has it zero and runs for Phase. Array children // are not stored individually: child i's state is derived from the parent // record, with ChildStops (keyed by the decimal index) holding any child that // was canceled or terminated. @@ -52,6 +53,7 @@ type jobRecord struct { Seq int64 `json:"seq"` SubmittedAt time.Time `json:"submittedAt"` Phase time.Duration `json:"phase"` + RunPhase time.Duration `json:"runPhase,omitempty"` Sequential bool `json:"sequential,omitempty"` Stop *jobStop `json:"stop,omitempty"` ChildStops map[string]jobStop `json:"childStops,omitempty"` @@ -118,6 +120,15 @@ func cloneJob(j driver.Job) driver.Job { return j } +// runDwell is how long the job stays RUNNING. +func (r *jobRecord) runDwell() time.Duration { + if r.RunPhase > 0 { + return r.RunPhase + } + + return r.Phase +} + func (r *jobRecord) arraySize() int { if r.Job.ArrayProperties == nil { return 0 @@ -379,7 +390,7 @@ func (o *observer) natural(rec *jobRecord, idx int, prev *jobView) jobView { return blocked } - return o.phaseView(ready, rec.Phase) + return o.phaseView(ready, rec.Phase, rec.runDwell()) } // readiness folds the job's dependencies (and a SEQUENTIAL predecessor) into @@ -422,10 +433,10 @@ func (o *observer) readiness(rec *jobRecord, idx int, prev *jobView, ready time. } // phaseView places a ready job in its RUNNABLE/STARTING/RUNNING/SUCCEEDED -// phase given the phase duration d. -func (o *observer) phaseView(ready time.Time, d time.Duration) jobView { +// phase given the phase duration d and the RUNNING dwell run. +func (o *observer) phaseView(ready time.Time, d, run time.Duration) jobView { started := ready.Add(phasesToRunning * d) - stopped := ready.Add(phasesToSucceeded * d) + stopped := started.Add(run) switch { case o.now.Before(ready.Add(phasesToStarting * d)): @@ -487,6 +498,32 @@ func withExitCode(container json.RawMessage) json.RawMessage { return out } +// withoutExitCode returns the container detail with any exitCode removed, for +// an event replaying a status the job held before its container exited. +func withoutExitCode(container json.RawMessage) json.RawMessage { + if !isSet(container) { + return container + } + + var obj map[string]json.RawMessage + if err := json.Unmarshal(container, &obj); err != nil { + return container + } + + if _, ok := obj["exitCode"]; !ok { + return container + } + + delete(obj, "exitCode") + + out, err := json.Marshal(obj) + if err != nil { + return container + } + + return out +} + // render builds the externally visible job (top-level, idx -1, or array child // idx) as observed by o. func (o *observer) render(rec *jobRecord, idx int) driver.Job { diff --git a/providers/aws/batch/job_queue.go b/providers/aws/batch/job_queue.go index 9be8a1b64..99506a0b8 100644 --- a/providers/aws/batch/job_queue.go +++ b/providers/aws/batch/job_queue.go @@ -50,6 +50,9 @@ func (m *Mock) CreateJobQueue(_ context.Context, in driver.CreateJobQueueInput) return nil, err } + m.policyBindMu.Lock() + defer m.policyBindMu.Unlock() + policyARN := "" if in.SchedulingPolicyARN != "" { @@ -103,6 +106,11 @@ func (m *Mock) UpdateJobQueue(_ context.Context, in driver.UpdateJobQueueInput) } } + // Held across the read, the policy resolution and the write, so the policy + // can't be deleted (and no other update can land) in between. + m.policyBindMu.Lock() + defer m.policyBindMu.Unlock() + cur, ok := m.jobQueues.Get(name) if !ok { return nil, cerrors.Newf(cerrors.NotFound, "job queue %q does not exist", name) diff --git a/providers/aws/batch/jobs.go b/providers/aws/batch/jobs.go index 49200f87e..1ad6ca11f 100644 --- a/providers/aws/batch/jobs.go +++ b/providers/aws/batch/jobs.go @@ -83,6 +83,7 @@ func (m *Mock) SubmitJob(ctx context.Context, in driver.SubmitJobInput) (*driver Seq: seq, SubmittedAt: now, Phase: m.opts.SettleDuration(settle.DefaultBatchJobPhase), + RunPhase: m.opts.SettleDuration(settle.DefaultBatchJobRunDwell), Sequential: hasSequential(in.DependsOn), LastEmitted: driver.JobStatusSubmitted, } @@ -260,7 +261,8 @@ func validateShare(q *driver.JobQueue, in *driver.SubmitJobInput) error { } // resolveJobDefinition resolves name, name:revision, or an ARN with or without -// a revision. Without a revision the latest ACTIVE revision is used. +// a revision. Without a revision the latest ACTIVE revision is used; an +// explicit revision must be ACTIVE. func (m *Mock) resolveJobDefinition(ref string) (driver.JobDefinition, error) { key := nameFromARN(kindJobDefinition, ref) @@ -270,6 +272,13 @@ func (m *Mock) resolveJobDefinition(ref string) (driver.JobDefinition, error) { return driver.JobDefinition{}, cerrors.Newf(cerrors.InvalidArgument, "job definition %q does not exist", ref) } + // A deregistered revision can't be submitted to, even by explicit + // revision. + if jd.Status != driver.JDStatusActive { + return driver.JobDefinition{}, cerrors.Newf(cerrors.InvalidArgument, + "job definition %q is %s and can't be used to submit jobs", ref, jd.Status) + } + return jd, nil } @@ -301,6 +310,11 @@ func (m *Mock) buildJob(in *driver.SubmitJobInput, q *driver.JobQueue, jd *drive return driver.Job{}, err } + props, err := applyPropertyOverrides(in, jd) + if err != nil { + return driver.Job{}, err + } + id := idgen.UUID() job := driver.Job{ @@ -312,9 +326,9 @@ func (m *Mock) buildJob(in *driver.SubmitJobInput, q *driver.JobQueue, jd *drive DependsOn: cloneDeps(in.DependsOn), Parameters: mergeParams(jd.Parameters, in.Parameters), Container: container, - NodeProperties: copyRaw(jd.NodeProperties), - EcsProperties: copyRaw(jd.EcsProperties), - EksProperties: copyRaw(jd.EksProperties), + NodeProperties: props.node, + EcsProperties: props.ecs, + EksProperties: props.eks, RetryStrategy: firstSet(in.RetryStrategy, jd.RetryStrategy), Timeout: firstSet(in.Timeout, jd.Timeout), PlatformCapabilities: copyStrings(jd.PlatformCapabilities), @@ -629,7 +643,7 @@ func (l *jobLister) queue(m *Mock) *driver.ListJobsOutput { } // children lists an array job's children by index, resuming after the cursor -// index. +// index. Filters apply to children the same way they apply to a queue listing. func (l *jobLister) children(m *Mock) *driver.ListJobsOutput { out := &driver.ListJobsOutput{Jobs: []driver.Job{}} @@ -640,7 +654,7 @@ func (l *jobLister) children(m *Mock) *driver.ListJobsOutput { for i := int(l.after + 1); i < rec.arraySize(); i++ { j := l.o.render(&rec, i) - if l.useStatus && j.Status != l.status { + if (l.useStatus && j.Status != l.status) || !matchesFilters(&j, l.in.Filters) { continue } diff --git a/providers/aws/batch/jobs_overrides_test.go b/providers/aws/batch/jobs_overrides_test.go new file mode 100644 index 000000000..af664d4ba --- /dev/null +++ b/providers/aws/batch/jobs_overrides_test.go @@ -0,0 +1,367 @@ +package batch_test + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "testing" + + "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/providers/aws/batch" + "github.com/stackshy/cloudemu/v2/services/batch/driver" +) + +func registerJD(t *testing.T, m *batch.Mock, in driver.RegisterJobDefinitionInput) { + t.Helper() + + if _, err := m.RegisterJobDefinition(context.Background(), in); err != nil { + t.Fatalf("register %s: %v", in.Name, err) + } +} + +func decodeMap(t *testing.T, raw json.RawMessage) map[string]any { + t.Helper() + + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatalf("decode %s: %v", raw, err) + } + + return out +} + +func TestSubmitJobAppliesEcsPropertiesOverride(t *testing.T) { + m := newMock(t) + setupQueue(t, m) + + registerJD(t, m, driver.RegisterJobDefinitionInput{ + Name: "ecs", Type: driver.JDTypeContainer, + EcsProperties: json.RawMessage(`{"taskProperties":[{"executionRoleArn":"role","containers":[ + {"name":"app","image":"app:1","command":["run"],"environment":[{"name":"A","value":"1"},{"name":"B","value":"2"}]}, + {"name":"side","image":"side:1","command":["tail"]}]}]}`), + }) + + j := submit(t, m, driver.SubmitJobInput{ + JobDefinition: "ecs", + EcsPropertiesOverride: json.RawMessage(`{"taskProperties":[{"containers":[ + {"name":"app","command":["run","--fast"],"environment":[{"name":"B","value":"20"},{"name":"C","value":"3"}]}]}]}`), + }) + + got := decodeMap(t, describe(t, m, j.JobID).EcsProperties) + task := got["taskProperties"].([]any)[0].(map[string]any) + + if task["executionRoleArn"] != "role" { + t.Fatalf("task-level properties lost: %v", task) + } + + containers := task["containers"].([]any) + app := containers[0].(map[string]any) + side := containers[1].(map[string]any) + + if fmt.Sprint(app["command"]) != "[run --fast]" || app["image"] != "app:1" { + t.Fatalf("app command not overridden or image lost: %v", app) + } + + if fmt.Sprint(app["environment"]) != "[map[name:A value:1] map[name:B value:20] map[name:C value:3]]" { + t.Fatalf("app environment not merged by name: %v", app["environment"]) + } + + if fmt.Sprint(side["command"]) != "[tail]" { + t.Fatalf("container that wasn't overridden changed: %v", side) + } + + // The job definition itself is untouched. + j2 := submit(t, m, driver.SubmitJobInput{JobDefinition: "ecs"}) + if c := decodeMap(t, describe(t, m, j2.JobID).EcsProperties)["taskProperties"].([]any)[0].(map[string]any)["containers"].([]any)[0].(map[string]any); fmt.Sprint(c["command"]) != "[run]" { + t.Fatalf("override leaked into the job definition: %v", c) + } +} + +func TestSubmitJobAppliesEksPropertiesOverride(t *testing.T) { + m := newMock(t) + setupQueue(t, m) + + registerJD(t, m, driver.RegisterJobDefinitionInput{ + Name: "eks", Type: driver.JDTypeContainer, + EksProperties: json.RawMessage(`{"podProperties":{"hostNetwork":true, + "containers":[{"name":"main","image":"img:1","command":["sleep"],"env":[{"name":"X","value":"1"}]}], + "metadata":{"labels":{"team":"a","env":"dev"}}}}`), + }) + + j := submit(t, m, driver.SubmitJobInput{ + JobDefinition: "eks", + EksPropertiesOverride: json.RawMessage(`{"podProperties":{ + "containers":[{"image":"img:2","args":["60"],"env":[{"name":"Y","value":"2"}]}], + "metadata":{"labels":{"env":"prod"}}}}`), + }) + + pod := decodeMap(t, describe(t, m, j.JobID).EksProperties)["podProperties"].(map[string]any) + c := pod["containers"].([]any)[0].(map[string]any) + + if c["image"] != "img:2" || fmt.Sprint(c["args"]) != "[60]" || fmt.Sprint(c["command"]) != "[sleep]" { + t.Fatalf("unnamed override on the only container not applied: %v", c) + } + + if fmt.Sprint(c["env"]) != "[map[name:X value:1] map[name:Y value:2]]" { + t.Fatalf("env not merged: %v", c["env"]) + } + + labels := pod["metadata"].(map[string]any)["labels"].(map[string]any) + if labels["team"] != "a" || labels["env"] != "prod" || pod["hostNetwork"] != true { + t.Fatalf("metadata labels not merged or pod settings lost: %v", pod) + } +} + +func TestSubmitJobAppliesNodeOverrides(t *testing.T) { + m := newMock(t) + setupQueue(t, m) + + registerJD(t, m, driver.RegisterJobDefinitionInput{ + Name: "mnp", Type: driver.JDTypeMultiNode, + NodeProperties: json.RawMessage(`{"numNodes":2,"mainNode":0,"nodeRangeProperties":[ + {"targetNodes":"0","container":{"image":"main","command":["lead"]}}, + {"targetNodes":"1:","container":{"image":"worker","command":["work"]}}]}`), + }) + + j := submit(t, m, driver.SubmitJobInput{ + JobDefinition: "mnp", + NodeOverrides: json.RawMessage(`{"numNodes":4,"nodePropertyOverrides":[ + {"targetNodes":"1:","containerOverrides":{"command":["work","--hard"]},"instanceTypes":["c5.large"]}]}`), + }) + + np := decodeMap(t, describe(t, m, j.JobID).NodeProperties) + if np["numNodes"] != float64(4) { + t.Fatalf("numNodes not overridden: %v", np) + } + + ranges := np["nodeRangeProperties"].([]any) + lead := ranges[0].(map[string]any)["container"].(map[string]any) + worker := ranges[1].(map[string]any) + + if fmt.Sprint(lead["command"]) != "[lead]" { + t.Fatalf("range outside targetNodes changed: %v", lead) + } + + if wc := worker["container"].(map[string]any); fmt.Sprint(wc["command"]) != "[work --hard]" || wc["image"] != "worker" { + t.Fatalf("worker container override not merged: %v", wc) + } + + if fmt.Sprint(worker["instanceTypes"]) != "[c5.large]" { + t.Fatalf("instanceTypes not overridden: %v", worker) + } +} + +func TestSubmitJobRejectsUnusableOverrides(t *testing.T) { + m := newMock(t) + setupQueue(t, m) + ctx := context.Background() + + registerJD(t, m, driver.RegisterJobDefinitionInput{ + Name: "ecs2", Type: driver.JDTypeContainer, + EcsProperties: json.RawMessage(`{"taskProperties":[{"containers":[{"name":"a","image":"a"},{"name":"b","image":"b"}]}]}`), + }) + registerJD(t, m, driver.RegisterJobDefinitionInput{ + Name: "mnp", Type: driver.JDTypeMultiNode, + NodeProperties: json.RawMessage(`{"numNodes":2,"mainNode":1,"nodeRangeProperties":[{"targetNodes":"0:1","container":{"image":"x"}}]}`), + }) + + cases := map[string]driver.SubmitJobInput{ + "ecs override on a containerProperties definition": {JobDefinition: "jd", + EcsPropertiesOverride: json.RawMessage(`{"taskProperties":[{"containers":[{"command":["x"]}]}]}`)}, + "eks override on an ECS definition": {JobDefinition: "ecs2", + EksPropertiesOverride: json.RawMessage(`{"podProperties":{"containers":[{"image":"x"}]}}`)}, + "node override on a container definition": {JobDefinition: "jd", + NodeOverrides: json.RawMessage(`{"numNodes":3}`)}, + "unknown container name": {JobDefinition: "ecs2", + EcsPropertiesOverride: json.RawMessage(`{"taskProperties":[{"containers":[{"name":"zzz","command":["x"]}]}]}`)}, + "unnamed override with two containers": {JobDefinition: "ecs2", + EcsPropertiesOverride: json.RawMessage(`{"taskProperties":[{"containers":[{"command":["x"]}]}]}`)}, + "numNodes without an open-ended range": {JobDefinition: "mnp", + NodeOverrides: json.RawMessage(`{"numNodes":3}`)}, + "targetNodes covering no range": {JobDefinition: "mnp", + NodeOverrides: json.RawMessage(`{"nodePropertyOverrides":[{"targetNodes":"5:6","containerOverrides":{"command":["x"]}}]}`)}, + } + + for name, in := range cases { + in.JobName, in.JobQueue = "j", "q" + if _, err := m.SubmitJob(ctx, in); !errors.IsInvalidArgument(err) { + t.Errorf("%s: want InvalidArgument, got %v", name, err) + } + } +} + +func TestSubmitJobRejectsInactiveRevision(t *testing.T) { + m := newMock(t) + setupQueue(t, m) + ctx := context.Background() + + registerJD(t, m, driver.RegisterJobDefinitionInput{ + Name: "jd", Type: driver.JDTypeContainer, ContainerProperties: json.RawMessage(`{"image":"v2"}`), + }) + + if err := m.DeregisterJobDefinition(ctx, "jd:1"); err != nil { + t.Fatal(err) + } + + for _, ref := range []string{"jd:1", "arn:aws:batch:us-east-1:000000000000:job-definition/jd:1"} { + _, err := m.SubmitJob(ctx, driver.SubmitJobInput{JobName: "j", JobQueue: "q", JobDefinition: ref}) + if !errors.IsInvalidArgument(err) { + t.Fatalf("submit with INACTIVE %s: want InvalidArgument, got %v", ref, err) + } + } + + // The ACTIVE revision still works by explicit revision. + submit(t, m, driver.SubmitJobInput{JobDefinition: "jd:2"}) +} + +func TestReplayedEventsCarryNoExitCode(t *testing.T) { + m := newMock(t) + pub := &recordingPublisher{} + m.SetEventPublisher(pub) + setupQueue(t, m) + + submit(t, m, driver.SubmitJobInput{}) + + pub.mu.Lock() + defer pub.mu.Unlock() + + if len(pub.events) != 4 { + t.Fatalf("want 4 events, got %d", len(pub.events)) + } + + for _, e := range pub.events { + c, _ := e.detail["container"].(map[string]any) + if c == nil || c["image"] != "busybox" { + t.Fatalf("%s event lost the container: %v", e.detail["status"], e.detail) + } + + _, hasExit := c["exitCode"] + if want := e.detail["status"] == driver.JobStatusSucceeded; hasExit != want { + t.Fatalf("%s event exitCode present = %v, want %v", e.detail["status"], hasExit, want) + } + } +} + +func TestListArrayChildrenAppliesFilters(t *testing.T) { + m := newMock(t) + setupQueue(t, m) + + size := int32(3) + parent := submit(t, m, driver.SubmitJobInput{JobName: "arr", ArraySize: &size}) + + nomatch, err := listJobs(m, driver.ListJobsInput{ + ArrayJobID: parent.JobID, + Filters: []driver.JobFilter{{Name: driver.FilterJobName, Values: []string{"nomatch"}}}, + }) + if err != nil || len(nomatch) != 0 { + t.Fatalf("non-matching JOB_NAME filter listed children: %v %v", nomatch, err) + } + + match, err := listJobs(m, driver.ListJobsInput{ + ArrayJobID: parent.JobID, + Filters: []driver.JobFilter{{Name: driver.FilterJobName, Values: []string{"ARR*"}}}, + }) + if err != nil || len(match) != 3 { + t.Fatalf("matching JOB_NAME filter: got %d children, %v", len(match), err) + } +} + +// TestSchedulingPolicyBindingIsAtomic races queue creation/update against +// policy deletion: no queue may end up bound to a deleted policy. +func TestSchedulingPolicyBindingIsAtomic(t *testing.T) { + m := newMock(t) + ctx := context.Background() + mustCreateCE(t, m, "ce") + + order := []driver.ComputeEnvironmentOrder{{Order: 1, ComputeEnvironment: "ce"}} + + for i := 0; i < 50; i++ { + pol := fmt.Sprintf("p%d", i) + spare := fmt.Sprintf("s%d", i) + + for _, n := range []string{pol, spare} { + if _, err := m.CreateSchedulingPolicy(ctx, driver.CreateSchedulingPolicyInput{Name: n}); err != nil { + t.Fatal(err) + } + } + + if _, err := m.CreateJobQueue(ctx, driver.CreateJobQueueInput{ + Name: "u" + pol, Priority: 1, ComputeEnvironmentOrder: order, SchedulingPolicyARN: spare, + }); err != nil { + t.Fatal(err) + } + + var wg sync.WaitGroup + + wg.Add(3) + + go func() { + defer wg.Done() + + _, _ = m.CreateJobQueue(ctx, driver.CreateJobQueueInput{ + Name: "c" + pol, Priority: 1, ComputeEnvironmentOrder: order, SchedulingPolicyARN: pol, + }) + }() + + go func() { + defer wg.Done() + + _, _ = m.UpdateJobQueue(ctx, driver.UpdateJobQueueInput{Name: "u" + pol, SchedulingPolicyARN: &pol}) + }() + + go func() { + defer wg.Done() + + _ = m.DeleteSchedulingPolicy(ctx, pol) + }() + + wg.Wait() + } + + queues, _ := m.DescribeJobQueues(ctx, nil) + for i := range queues { + q := &queues[i] + if q.SchedulingPolicyARN == "" { + continue + } + + if got, _ := m.DescribeSchedulingPolicies(ctx, []string{q.SchedulingPolicyARN}); len(got) != 1 { + t.Fatalf("queue %s is bound to deleted policy %s", q.Name, q.SchedulingPolicyARN) + } + } +} + +func TestMultinodeJobDefinitionEchoesNoDefaultPlatform(t *testing.T) { + m := newMock(t) + ctx := context.Background() + + registerJD(t, m, driver.RegisterJobDefinitionInput{ + Name: "mnp", Type: driver.JDTypeMultiNode, + NodeProperties: json.RawMessage(`{"numNodes":2,"mainNode":0,"nodeRangeProperties":[{"targetNodes":"0:","container":{"image":"x"}}]}`), + }) + registerJD(t, m, driver.RegisterJobDefinitionInput{ + Name: "fg", Type: driver.JDTypeContainer, PlatformCapabilities: []string{"FARGATE"}, + ContainerProperties: json.RawMessage(`{"image":"x"}`), + }) + + defs, err := m.DescribeJobDefinitions(ctx, driver.DescribeJobDefinitionsInput{}) + if err != nil || len(defs) != 2 { + t.Fatalf("describe: %v %v", defs, err) + } + + for i := range defs { + d := &defs[i] + + switch d.Name { + case "mnp": + if len(d.PlatformCapabilities) != 0 { + t.Fatalf("multinode definition echoes a default platform: %v", d.PlatformCapabilities) + } + case "fg": + if fmt.Sprint(d.PlatformCapabilities) != "[FARGATE]" { + t.Fatalf("explicit platform lost: %v", d.PlatformCapabilities) + } + } + } +} diff --git a/providers/aws/batch/jobs_test.go b/providers/aws/batch/jobs_test.go index 04b44a33b..880927ee5 100644 --- a/providers/aws/batch/jobs_test.go +++ b/providers/aws/batch/jobs_test.go @@ -9,6 +9,7 @@ import ( "github.com/stackshy/cloudemu/v2/config" "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/settle" "github.com/stackshy/cloudemu/v2/providers/aws/batch" "github.com/stackshy/cloudemu/v2/services/batch/driver" ) @@ -165,26 +166,38 @@ func TestJobLifecycleAdvancesWithClock(t *testing.T) { j := submit(t, m, driver.SubmitJobInput{}) - want := []string{ - driver.JobStatusSubmitted, driver.JobStatusRunnable, driver.JobStatusStarting, - driver.JobStatusRunning, driver.JobStatusSucceeded, - } - - for i, w := range want { + // One second each in SUBMITTED, RUNNABLE and STARTING; RUNNING lasts the + // longer run dwell, so a real client has time to terminate a running job. + steps := []struct { + status string + dwell time.Duration + }{ + {driver.JobStatusSubmitted, time.Second}, + {driver.JobStatusRunnable, time.Second}, + {driver.JobStatusStarting, time.Second}, + {driver.JobStatusRunning, time.Second}, + {driver.JobStatusRunning, settle.DefaultBatchJobRunDwell - 2*time.Second}, + {driver.JobStatusRunning, time.Second}, + {driver.JobStatusSucceeded, 0}, + } + + for i, st := range steps { got := describe(t, m, j.JobID) - if got.Status != w { - t.Fatalf("step %d: status = %s, want %s", i, got.Status, w) + if got.Status != st.status { + t.Fatalf("step %d: status = %s, want %s", i, got.Status, st.status) } - if w == driver.JobStatusRunning && got.StartedAt == 0 { + if st.status == driver.JobStatusRunning && got.StartedAt == 0 { t.Fatal("RUNNING job has no startedAt") } - if w == driver.JobStatusSucceeded && (got.StoppedAt <= got.StartedAt || got.StatusReason == "") { - t.Fatalf("SUCCEEDED job times/reason wrong: %+v", got) + if st.status == driver.JobStatusSucceeded { + if got.StoppedAt-got.StartedAt != settle.DefaultBatchJobRunDwell.Milliseconds() || got.StatusReason == "" { + t.Fatalf("SUCCEEDED job times/reason wrong: %+v", got) + } } - fc.Advance(time.Second) + fc.Advance(st.dwell) } } @@ -268,15 +281,15 @@ func TestDependentJobWaitsAndFailsWithDependency(t *testing.T) { t.Fatalf("child before parent finishes = %s, want PENDING", got) } - // Parent finishes at +4s; child then runs RUNNABLE/STARTING/RUNNING and - // succeeds at +7s. - fc.Advance(3 * time.Second) + // Parent finishes at +8s (3 one-second phases plus the 5s RUNNING dwell); + // child then runs RUNNABLE/STARTING/RUNNING and succeeds at +15s. + fc.Advance(7 * time.Second) if got := describe(t, m, child.JobID).Status; got != driver.JobStatusRunnable { t.Fatalf("child after parent succeeded = %s, want RUNNABLE", got) } - fc.Advance(3 * time.Second) + fc.Advance(7 * time.Second) if got := describe(t, m, child.JobID).Status; got != driver.JobStatusSucceeded { t.Fatalf("child = %s, want SUCCEEDED", got) @@ -713,12 +726,18 @@ func TestJobsAndPoliciesSurviveSnapshot(t *testing.T) { t.Fatalf("restored job = %+v", got) } - fc.Advance(5 * time.Second) + fc.Advance(7 * time.Second) - if got := describe(t, dst, j.JobID); got.Status != driver.JobStatusSucceeded { + got := describe(t, dst, j.JobID) + if got.Status != driver.JobStatusSucceeded { t.Fatalf("restored job did not resume its lifecycle: %s", got.Status) } + // The RUNNING dwell is part of the restored record. + if got.StoppedAt-got.StartedAt != settle.DefaultBatchJobRunDwell.Milliseconds() { + t.Fatalf("restored job ran for %dms, want %v", got.StoppedAt-got.StartedAt, settle.DefaultBatchJobRunDwell) + } + if list, _ := dst.ListSchedulingPolicies(ctx); len(list) != 1 { t.Fatalf("scheduling policy not restored: %+v", list) } diff --git a/providers/aws/batch/overrides.go b/providers/aws/batch/overrides.go new file mode 100644 index 000000000..98e363404 --- /dev/null +++ b/providers/aws/batch/overrides.go @@ -0,0 +1,609 @@ +package batch + +import ( + "encoding/json" + "strconv" + "strings" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/services/batch/driver" +) + +// jsonObject is a decoded JSON object whose members stay raw, so fields the +// emulator doesn't interpret are echoed byte for byte. +type jsonObject = map[string]json.RawMessage + +// jobProperties is the node/ECS/EKS detail of a new job: the job definition's +// blobs with the SubmitJob overrides applied. +type jobProperties struct { + node, ecs, eks json.RawMessage +} + +// applyPropertyOverrides applies SubmitJob nodeOverrides, ecsPropertiesOverride +// and eksPropertiesOverride to the job definition. Each override is only valid +// for a job definition that has the matching properties (SubmitJob API +// reference): ECS overrides need ecsProperties, EKS overrides eksProperties, +// and node overrides a multinode job definition. +func applyPropertyOverrides(in *driver.SubmitJobInput, jd *driver.JobDefinition) (jobProperties, error) { + p := jobProperties{node: copyRaw(jd.NodeProperties), ecs: copyRaw(jd.EcsProperties), eks: copyRaw(jd.EksProperties)} + + var err error + + if isSet(in.EcsPropertiesOverride) { + if !isSet(jd.EcsProperties) { + return p, cerrors.New(cerrors.InvalidArgument, + "ecsPropertiesOverride can only be specified for a job definition with ecsProperties") + } + + if p.ecs, err = mergeEcsProperties(jd.EcsProperties, in.EcsPropertiesOverride); err != nil { + return p, err + } + } + + if isSet(in.EksPropertiesOverride) { + if !isSet(jd.EksProperties) { + return p, cerrors.New(cerrors.InvalidArgument, + "eksPropertiesOverride can only be specified for a job definition with eksProperties") + } + + if p.eks, err = mergeEksProperties(jd.EksProperties, in.EksPropertiesOverride); err != nil { + return p, err + } + } + + if isSet(in.NodeOverrides) { + if !isSet(jd.NodeProperties) { + return p, cerrors.New(cerrors.InvalidArgument, + "nodeOverrides can only be specified for a multinode job definition") + } + + if p.node, err = mergeNodeProperties(jd.NodeProperties, in.NodeOverrides); err != nil { + return p, err + } + } + + return p, nil +} + +// decodeObject decodes a JSON object; an unset value is an empty object. +func decodeObject(raw json.RawMessage, what string) (jsonObject, error) { + obj := jsonObject{} + if !isSet(raw) { + return obj, nil + } + + if err := json.Unmarshal(raw, &obj); err != nil { + return nil, cerrors.Newf(cerrors.InvalidArgument, "invalid %s: %v", what, err) + } + + return obj, nil +} + +// decodeObjects decodes a JSON array of objects; an unset value is empty. +func decodeObjects(raw json.RawMessage, what string) ([]jsonObject, error) { + if !isSet(raw) { + return nil, nil + } + + var out []jsonObject + if err := json.Unmarshal(raw, &out); err != nil { + return nil, cerrors.Newf(cerrors.InvalidArgument, "invalid %s: %v", what, err) + } + + return out, nil +} + +func encodeJSON(v any, what string) (json.RawMessage, error) { + out, err := json.Marshal(v) + if err != nil { + return nil, cerrors.Newf(cerrors.Internal, "encode %s: %v", what, err) + } + + return out, nil +} + +// mergeFields applies the set members of over onto base: members named in +// keyed are arrays of objects merged by the given key field (mergeKeyed), +// members in maps are string maps merged key by key, and every other member +// replaces the base value. +func mergeFields(base, over jsonObject, keyed map[string]string, maps map[string]bool) { + for k, v := range over { + if !isSet(v) { + continue + } + + switch { + case keyed[k] != "": + base[k] = mergeKeyed(base[k], v, keyed[k]) + case maps[k]: + base[k] = mergeStringMap(base[k], v) + default: + base[k] = v + } + } +} + +// mergeStringMap merges two JSON string maps, over winning on a shared key. +func mergeStringMap(base, over json.RawMessage) json.RawMessage { + var b, o map[string]json.RawMessage + + if err := json.Unmarshal(over, &o); err != nil { + return over + } + + if isSet(base) { + if err := json.Unmarshal(base, &b); err != nil { + return over + } + } + + if b == nil { + b = make(map[string]json.RawMessage, len(o)) + } + + for k, v := range o { + b[k] = v + } + + out, err := json.Marshal(b) + if err != nil { + return over + } + + return out +} + +// containerName returns the "name" member of a container object. +func containerName(c jsonObject) string { + var s string + + if raw, ok := c["name"]; ok { + _ = json.Unmarshal(raw, &s) + } + + return s +} + +// mergeNamedContainers applies container overrides to a container list. Each +// override names the container it replaces settings of; an unnamed override is +// accepted only when the list has exactly one container. +func mergeNamedContainers( + base []jsonObject, overrides []jsonObject, what string, keyed map[string]string, +) ([]jsonObject, error) { + for _, o := range overrides { + name := containerName(o) + + target := -1 + + switch { + case name == "" && len(base) == 1: + target = 0 + case name == "": + return nil, cerrors.Newf(cerrors.InvalidArgument, + "%s: name is required when the job definition has more than one container", what) + default: + for i := range base { + if containerName(base[i]) == name { + target = i + + break + } + } + } + + if target < 0 { + return nil, cerrors.Newf(cerrors.InvalidArgument, + "%s: container %q is not defined in the job definition", what, name) + } + + mergeFields(base[target], o, keyed, nil) + } + + return base, nil +} + +//nolint:gochecknoglobals // immutable merge rules, read-only after init +var ( + // ECS TaskContainerOverrides: environment merges by name, resource + // requirements by type; command (and name) replace. + ecsContainerKeyed = map[string]string{"environment": "name", "resourceRequirements": "type"} + // EksContainerOverride: env merges by name; image, command, args and + // resources replace. + eksContainerKeyed = map[string]string{"env": "name"} + // EksMetadata: labels and annotations merge key by key; namespace replaces. + eksMetadataMaps = map[string]bool{"labels": true, "annotations": true} +) + +// mergeEcsProperties applies an EcsPropertiesOverride: taskProperties[i] +// overrides the definition's taskProperties[i], each container override +// targeting a container by name. +func mergeEcsProperties(props, override json.RawMessage) (json.RawMessage, error) { + const what = "ecsPropertiesOverride" + + base, err := decodeObject(props, "ecsProperties") + if err != nil { + return copyRaw(props), nil //nolint:nilerr // a non-object definition blob is echoed unchanged + } + + over, err := decodeObject(override, what) + if err != nil { + return nil, err + } + + baseTasks, err := decodeObjects(base["taskProperties"], "ecsProperties.taskProperties") + if err != nil { + return copyRaw(props), nil //nolint:nilerr // a non-object definition blob is echoed unchanged + } + + overTasks, err := decodeObjects(over["taskProperties"], what+".taskProperties") + if err != nil { + return nil, err + } + + if len(overTasks) > len(baseTasks) { + return nil, cerrors.Newf(cerrors.InvalidArgument, + "%s lists %d taskProperties but the job definition has %d", what, len(overTasks), len(baseTasks)) + } + + for i, ot := range overTasks { + if mergeErr := mergeTaskContainers(baseTasks[i], ot, "containers", what, ecsContainerKeyed); mergeErr != nil { + return nil, mergeErr + } + } + + if len(overTasks) > 0 { + if base["taskProperties"], err = encodeJSON(baseTasks, "ecsProperties"); err != nil { + return nil, err + } + } + + return encodeJSON(base, "ecsProperties") +} + +// mergeTaskContainers merges the container overrides listed under field of +// over into the same field of base. +func mergeTaskContainers(base, over jsonObject, field, what string, keyed map[string]string) error { + overs, err := decodeObjects(over[field], what+"."+field) + if err != nil { + return err + } + + if len(overs) == 0 { + return nil + } + + containers, err := decodeObjects(base[field], field) + if err != nil { + return cerrors.Newf(cerrors.InvalidArgument, "%s: the job definition's %s can't be overridden", what, field) + } + + merged, err := mergeNamedContainers(containers, overs, what+"."+field, keyed) + if err != nil { + return err + } + + base[field], err = encodeJSON(merged, field) + + return err +} + +// mergeEksProperties applies an EksPropertiesOverride to the definition's +// podProperties: containers and initContainers by name, plus pod metadata. +func mergeEksProperties(props, override json.RawMessage) (json.RawMessage, error) { + const what = "eksPropertiesOverride" + + base, err := decodeObject(props, "eksProperties") + if err != nil { + return copyRaw(props), nil //nolint:nilerr // a non-object definition blob is echoed unchanged + } + + over, err := decodeObject(override, what) + if err != nil { + return nil, err + } + + overPod, err := decodeObject(over["podProperties"], what+".podProperties") + if err != nil { + return nil, err + } + + if len(overPod) == 0 { + return copyRaw(props), nil + } + + pod, err := decodeObject(base["podProperties"], "eksProperties.podProperties") + if err != nil { + return copyRaw(props), nil //nolint:nilerr // a non-object definition blob is echoed unchanged + } + + if podErr := mergeEksPod(pod, overPod); podErr != nil { + return nil, podErr + } + + if base["podProperties"], err = encodeJSON(pod, "podProperties"); err != nil { + return nil, err + } + + return encodeJSON(base, "eksProperties") +} + +// mergeEksPod applies an EksPodPropertiesOverride to podProperties: containers +// and initContainers by name, plus pod metadata. +func mergeEksPod(pod, overPod jsonObject) error { + const what = "eksPropertiesOverride.podProperties" + + for _, field := range []string{"containers", "initContainers"} { + if err := mergeTaskContainers(pod, overPod, field, what, eksContainerKeyed); err != nil { + return err + } + } + + if !isSet(overPod["metadata"]) { + return nil + } + + meta, err := mergeEksMetadata(pod["metadata"], overPod["metadata"]) + if err != nil { + return err + } + + pod["metadata"] = meta + + return nil +} + +// mergeEksMetadata applies an EksMetadata override: labels and annotations +// merge key by key, namespace replaces. +func mergeEksMetadata(base, override json.RawMessage) (json.RawMessage, error) { + meta, err := decodeObject(base, "eksProperties.podProperties.metadata") + if err != nil { + meta = jsonObject{} + } + + overMeta, err := decodeObject(override, "eksPropertiesOverride.podProperties.metadata") + if err != nil { + return nil, err + } + + mergeFields(meta, overMeta, nil, eksMetadataMaps) + + return encodeJSON(meta, "metadata") +} + +// nodeRange is a resolved targetNodes range, both ends inclusive. +type nodeRange struct{ start, end int } + +// parseTargetNodes resolves a targetNodes value ("0:3", ":3", "2:", ":" or a +// single index) against the job's node count: an omitted start is 0 and an +// omitted end the highest node index. +func parseTargetNodes(s string, numNodes int) (nodeRange, bool) { + lo, hi, isRange := strings.Cut(s, ":") + if !isRange { + hi = lo + } + + r := nodeRange{start: 0, end: numNodes - 1} + + if lo != "" { + n, err := strconv.Atoi(lo) + if err != nil || n < 0 { + return nodeRange{}, false + } + + r.start = n + } + + if hi != "" { + n, err := strconv.Atoi(hi) + if err != nil || n < 0 { + return nodeRange{}, false + } + + r.end = n + } + + return r, r.start <= r.end +} + +// openUpper reports whether a targetNodes value has no upper bound ("n:" or +// ":"), and returns its lower bound. +func openUpper(s string) (lower int, open bool) { + lo, hi, isRange := strings.Cut(s, ":") + if !isRange || hi != "" { + return 0, false + } + + if lo == "" { + return 0, true + } + + n, err := strconv.Atoi(lo) + if err != nil { + return 0, false + } + + return n, true +} + +// nodeOverrides is the NodeOverrides shape; each property override stays raw +// apart from its target range. +type nodeOverrides struct { + NumNodes *int `json:"numNodes"` + NodePropertyOverrides []jsonObject `json:"nodePropertyOverrides"` +} + +// nodeDefinition is the part of NodeProperties a node override is checked +// against. +type nodeDefinition struct { + NumNodes int `json:"numNodes"` + MainNode int `json:"mainNode"` + NodeRangeProperties []jsonObject `json:"nodeRangeProperties"` +} + +// mergeNodeProperties applies NodeOverrides: numNodes replaces the definition's +// node count under the documented conditions, and each nodePropertyOverride +// applies to every node range of the definition that lies inside its +// targetNodes. +func mergeNodeProperties(props, override json.RawMessage) (json.RawMessage, error) { + const what = "nodeOverrides" + + var no nodeOverrides + if err := json.Unmarshal(override, &no); err != nil { + return nil, cerrors.Newf(cerrors.InvalidArgument, "invalid %s: %v", what, err) + } + + base, err := decodeObject(props, "nodeProperties") + if err != nil { + return copyRaw(props), nil //nolint:nilerr // a non-object definition blob is echoed unchanged + } + + var def nodeDefinition + if json.Unmarshal(props, &def) != nil { + return copyRaw(props), nil + } + + if no.NumNodes != nil { + if numErr := checkNumNodesOverride(&def, *no.NumNodes); numErr != nil { + return nil, numErr + } + + def.NumNodes = *no.NumNodes + base["numNodes"] = json.RawMessage(strconv.Itoa(*no.NumNodes)) + } + + for i, po := range no.NodePropertyOverrides { + if poErr := applyNodePropertyOverride(&def, po, i); poErr != nil { + return nil, poErr + } + } + + if len(no.NodePropertyOverrides) > 0 { + if base["nodeRangeProperties"], err = encodeJSON(def.NodeRangeProperties, "nodeRangeProperties"); err != nil { + return nil, err + } + } + + return encodeJSON(base, "nodeProperties") +} + +// checkNumNodesOverride enforces the NodeOverrides.numNodes conditions: the +// definition has a node range with an open upper boundary whose lower boundary +// is below the new count, and the main node index is below it too. +func checkNumNodesOverride(def *nodeDefinition, n int) error { + if n < 1 { + return cerrors.Newf(cerrors.InvalidArgument, "nodeOverrides.numNodes must be at least 1, got %d", n) + } + + if def.MainNode >= n { + return cerrors.Newf(cerrors.InvalidArgument, + "nodeOverrides.numNodes (%d) must be greater than the job definition's mainNode (%d)", n, def.MainNode) + } + + for _, r := range def.NodeRangeProperties { + var target string + _ = json.Unmarshal(r["targetNodes"], &target) + + if lower, open := openUpper(target); open && lower < n { + return nil + } + } + + return cerrors.Newf(cerrors.InvalidArgument, + "nodeOverrides.numNodes requires a node range with an open upper boundary (such as \"n:\") "+ + "whose lower boundary is below %d", n) +} + +// nodeRangeOverrideFields says how each NodePropertyOverride member maps onto a +// NodeRangeProperty member. +// +//nolint:gochecknoglobals // immutable mapping, read-only after init +var nodeRangeOverrideFields = map[string]string{ + "containerOverrides": "container", + "ecsPropertiesOverride": "ecsProperties", + "eksPropertiesOverride": "eksProperties", + "instanceTypes": "instanceTypes", + "consumableResourcePropertiesOverride": "consumableResourceProperties", +} + +// applyNodePropertyOverride applies one NodePropertyOverride to every node +// range of def inside its targetNodes. An override that covers no node range of +// the job definition is rejected rather than ignored. +func applyNodePropertyOverride(def *nodeDefinition, po jsonObject, i int) error { + var target string + _ = json.Unmarshal(po["targetNodes"], &target) + + if target == "" { + return cerrors.Newf(cerrors.InvalidArgument, "nodeOverrides.nodePropertyOverrides[%d].targetNodes is required", i) + } + + or, ok := parseTargetNodes(target, def.NumNodes) + if !ok { + return cerrors.Newf(cerrors.InvalidArgument, + "nodeOverrides.nodePropertyOverrides[%d].targetNodes %q is not a valid node range", i, target) + } + + matched := false + + for _, r := range def.NodeRangeProperties { + var rt string + _ = json.Unmarshal(r["targetNodes"], &rt) + + rr, ok := parseTargetNodes(rt, def.NumNodes) + if !ok || rr.start < or.start || rr.end > or.end { + continue + } + + if err := overrideNodeRange(r, po); err != nil { + return err + } + + matched = true + } + + if !matched { + return cerrors.Newf(cerrors.InvalidArgument, + "nodeOverrides.nodePropertyOverrides[%d].targetNodes %q doesn't cover any node range of the job definition", + i, target) + } + + return nil +} + +// overrideNodeRange applies one NodePropertyOverride to one node range. +func overrideNodeRange(r, po jsonObject) error { + for from, to := range nodeRangeOverrideFields { + v := po[from] + if !isSet(v) { + continue + } + + var ( + merged json.RawMessage + err error + ) + + switch from { + case "containerOverrides": + if !isSet(r[to]) { + return cerrors.New(cerrors.InvalidArgument, + "nodeOverrides: containerOverrides target a node range without container properties") + } + + merged, err = mergeContainer(r[to], v) + case "ecsPropertiesOverride": + merged, err = mergeEcsProperties(r[to], v) + case "eksPropertiesOverride": + merged, err = mergeEksProperties(r[to], v) + default: + merged = copyRaw(v) + } + + if err != nil { + return err + } + + r[to] = merged + } + + return nil +} diff --git a/providers/aws/batch/scheduling_policy.go b/providers/aws/batch/scheduling_policy.go index 86ae1733b..046b9a8f9 100644 --- a/providers/aws/batch/scheduling_policy.go +++ b/providers/aws/batch/scheduling_policy.go @@ -250,6 +250,9 @@ func (m *Mock) DeleteSchedulingPolicy(_ context.Context, arn string) error { name := nameFromARN(kindSchedulingPolicy, arn) + m.policyBindMu.Lock() + defer m.policyBindMu.Unlock() + p, ok := m.schedPolicies.Get(name) if !ok { return cerrors.Newf(cerrors.NotFound, "scheduling policy %q does not exist", arn) diff --git a/server/aws/batch/jobs_sdk_test.go b/server/aws/batch/jobs_sdk_test.go index 778c1d327..10f29f568 100644 --- a/server/aws/batch/jobs_sdk_test.go +++ b/server/aws/batch/jobs_sdk_test.go @@ -238,3 +238,54 @@ func TestSDKRegisterJobDefinitionInvalidType(t *testing.T) { }) requireClientException(t, err, "RegisterJobDefinition multinode without nodeProperties") } + +// TestSDKSubmitJobEcsPropertiesOverride checks an ECS override round-trips +// through the wire layer into DescribeJobs instead of being dropped. +func TestSDKSubmitJobEcsPropertiesOverride(t *testing.T) { + ctx := context.Background() + c := newBatchClient(t) + setupSDKQueue(t, c, nil) + + if _, err := c.RegisterJobDefinition(ctx, &awsbatch.RegisterJobDefinitionInput{ + JobDefinitionName: aws.String("ecs"), + Type: batchtypes.JobDefinitionTypeContainer, + EcsProperties: &batchtypes.EcsProperties{TaskProperties: []batchtypes.EcsTaskProperties{{ + Containers: []batchtypes.TaskContainerProperties{{Name: aws.String("app"), Image: aws.String("app:1"), Command: []string{"run"}}}, + }}}, + }); err != nil { + t.Fatalf("RegisterJobDefinition: %v", err) + } + + sub, err := c.SubmitJob(ctx, &awsbatch.SubmitJobInput{ + JobName: aws.String("ecs"), JobQueue: aws.String("q"), JobDefinition: aws.String("ecs"), + EcsPropertiesOverride: &batchtypes.EcsPropertiesOverride{TaskProperties: []batchtypes.TaskPropertiesOverride{{ + Containers: []batchtypes.TaskContainerOverrides{{Name: aws.String("app"), Command: []string{"run", "--fast"}}}, + }}}, + }) + if err != nil { + t.Fatalf("SubmitJob: %v", err) + } + + desc, err := c.DescribeJobs(ctx, &awsbatch.DescribeJobsInput{Jobs: []string{aws.ToString(sub.JobId)}}) + if err != nil || len(desc.Jobs) != 1 { + t.Fatalf("DescribeJobs: %v %v", desc, err) + } + + ep := desc.Jobs[0].EcsProperties + if ep == nil || len(ep.TaskProperties) != 1 || len(ep.TaskProperties[0].Containers) != 1 { + t.Fatalf("ecsProperties missing: %+v", ep) + } + + if got := ep.TaskProperties[0].Containers[0].Command; len(got) != 2 || got[1] != "--fast" { + t.Fatalf("override not applied: %v", got) + } + + // An ECS override on a containerProperties definition is a ClientException. + _, err = c.SubmitJob(ctx, &awsbatch.SubmitJobInput{ + JobName: aws.String("bad"), JobQueue: aws.String("q"), JobDefinition: aws.String("jd"), + EcsPropertiesOverride: &batchtypes.EcsPropertiesOverride{TaskProperties: []batchtypes.TaskPropertiesOverride{{ + Containers: []batchtypes.TaskContainerOverrides{{Command: []string{"x"}}}, + }}}, + }) + requireClientException(t, err, "ecs override on containerProperties definition") +}