Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ jobs:
env:
TOKENIZERS_PARALLELISM: 'false'
OMP_NUM_THREADS: '2'
GLOO_SOCKET_IFNAME: ${{ matrix.os == 'macos-15' && 'lo0' || 'lo' }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
Expand All @@ -40,8 +41,10 @@ jobs:
run: |
for iteration in $(seq 1 50); do
echo "CPU owner/fixture repetition $iteration/50"
uv run --no-sync python -m pytest -q tests/test_cache_retirement.py \
uv run --no-sync python -m pytest -q tests/test_cache_retirement.py tests/test_execution_retirement.py tests/test_cache_identity.py \
tests/test_distributed_http.py::test_rank_fixture_joins_watcher_before_destroying_process_groups \
tests/test_distributed_http.py::test_failed_runner_retention_does_not_leave_fixture_gloo_threads_alive \
tests/test_distributed_http.py::test_distributed_http_routes_real_replica_engines[fatal-2] \
tests/test_deepseek_auxiliary_graphs.py::test_auxiliary_callbacks_bind_model_projections_and_preserve_output_dtypes \
models/cuda/tests/test_deepseek_v41_projections.py::test_prepared_vocab_head_preserves_last_row_full_logits_and_fp32_output
done
25 changes: 25 additions & 0 deletions crates/orbitinfer-core/src/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ use crate::{
Phase, PlanningConfig, PlanningError, RequestId, RequestProgress, Schedulable, Scheduler,
Selection, SequenceStep, TokenRange, schedule_ready_prefix,
};
use std::any::Any;
use std::cmp::Reverse;
use std::fmt;
use std::num::{NonZeroU32, NonZeroU64, NonZeroUsize};
Expand Down Expand Up @@ -546,6 +547,7 @@ struct Batch<P> {
}

pub struct EngineCore<P> {
backing_owner: Option<Arc<dyn Any + Send + Sync>>,
metrics: Option<EngineMetricsState>,
in_flight: Option<Batch<P>>,
continuation: Option<ExecutionPlan<P>>,
Expand All @@ -567,13 +569,35 @@ pub struct EngineCore<P> {
closed: bool,
}

impl<P> Drop for EngineCore<P> {
fn drop(&mut self) {
if self.in_flight.is_some() || self.quarantined.is_some() || self.poisoned_error.is_some() {
// Drop is not a device receipt. Retain actual backing together with
// the bounded batch, logical page charges and storage borrowers.
// No cleanup is retried after an uncertain native failure.
std::mem::forget(self.backing_owner.take());
std::mem::forget(self.in_flight.take());
std::mem::forget(self.quarantined.take());
std::mem::forget(self.continuation.take());
std::mem::forget(self.kv_resources.take());
}
}
}

impl<P: Clone> EngineCore<P> {
/// Bind the actual arena/runtime owner to the logical page allocator.
///
/// `backing_owner` must retain every allocation and runtime resource the
/// runner can access, including allocations installed by resource binding.
/// Prepared work and uncertain failure keep this owner even after core drop.
/// Complete every accepted step and call `close` for orderly retirement.
pub fn new(
runner_config: RunnerConfig,
max_running_requests: usize,
context_length: u32,
scheduler_config: SchedulerConfig,
prefix_cache: bool,
backing_owner: Arc<dyn Any + Send + Sync>,
) -> Result<Self> {
let max_running_requests = NonZeroUsize::new(max_running_requests)
.ok_or_else(|| Error::invalid("max_running_requests must be positive"))?;
Expand Down Expand Up @@ -617,6 +641,7 @@ impl<P: Clone> EngineCore<P> {
let new_token_ratio_decay =
(new_token_ratio - minimum_new_token_ratio) / NEW_TOKEN_RATIO_DECAY_STEPS;
Ok(Self {
backing_owner: Some(backing_owner),
metrics: None,
in_flight: None,
continuation: None,
Expand Down
79 changes: 79 additions & 0 deletions crates/orbitinfer-core/tests/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ fn engine(lookahead: u32, chunk: u32) -> EngineCore<&'static str> {
..Default::default()
},
false,
std::sync::Arc::new(()),
)
.unwrap()
}
Expand Down Expand Up @@ -125,6 +126,7 @@ fn admission_credits_restored_physical_prefix_pages_only() {
..Default::default()
},
true,
std::sync::Arc::new(()),
)
.unwrap();
engine.enable_metrics().unwrap();
Expand Down Expand Up @@ -213,6 +215,7 @@ fn check_stopped_speculative_prefix(stop_token: bool) {
32,
SchedulerConfig::default(),
true,
std::sync::Arc::new(()),
)
.unwrap();
let mut model = Model {
Expand Down Expand Up @@ -531,6 +534,7 @@ fn panicking_checkpoint_access_quarantines_the_accepted_transaction() {
128,
SchedulerConfig::default(),
false,
std::sync::Arc::new(()),
)
.unwrap()
};
Expand Down Expand Up @@ -584,6 +588,7 @@ fn malformed_output_and_receipt_panic_quarantine_before_any_result_is_published(
128,
SchedulerConfig::default(),
false,
std::sync::Arc::new(()),
)
.unwrap()
};
Expand Down Expand Up @@ -664,6 +669,7 @@ fn a_panicking_payload_clone_restores_the_cohort_and_releases_its_cache_access()
128,
SchedulerConfig::default(),
false,
std::sync::Arc::new(()),
)
.unwrap();
let mut handles = Vec::new();
Expand Down Expand Up @@ -807,6 +813,7 @@ fn decode_continuation_transfers_writer_lease_and_commits_exact_feedback_depth()
512,
SchedulerConfig::default(),
true,
std::sync::Arc::new(()),
)
.unwrap();
let mut model = Model::default();
Expand Down Expand Up @@ -868,6 +875,7 @@ fn unsafe_decode_boundaries_fall_back_before_enqueuing_a_continuation() {
512,
SchedulerConfig::default(),
false,
std::sync::Arc::new(()),
)
.unwrap();
let mut model = Model::default();
Expand Down Expand Up @@ -916,6 +924,7 @@ fn cross_page_engine(prefix_cache: bool) -> EngineCore<&'static str> {
256,
SchedulerConfig::default(),
prefix_cache,
std::sync::Arc::new(()),
)
.unwrap()
}
Expand Down Expand Up @@ -1029,6 +1038,7 @@ fn shifted_decode_engine() -> Result<EngineCore<&'static str>, Error> {
128,
SchedulerConfig::default(),
false,
std::sync::Arc::new(()),
)
}

Expand Down Expand Up @@ -1089,6 +1099,7 @@ fn accepted_limit_defaults_to_one_and_rejects_zero_or_overflowing_write_footprin
128,
SchedulerConfig::default(),
false,
std::sync::Arc::new(()),
);
assert_eq!(result.err().unwrap().kind, ErrorKind::InvalidInput);
}
Expand All @@ -1114,6 +1125,7 @@ fn shifted_write_guard_survives_a_near_context_limit_without_changing_outputs()
128,
SchedulerConfig::default(),
false,
std::sync::Arc::new(()),
)
.unwrap();
assert!(engine.submit(input(vec![1; 126], 4)).is_err());
Expand Down Expand Up @@ -1210,6 +1222,7 @@ fn window_guard_bounds_automatic_reservation_and_rejects_an_oversized_write() {
256,
SchedulerConfig::default(),
false,
std::sync::Arc::new(()),
);
if read_window == 9 {
assert_eq!(result.err().unwrap().kind, ErrorKind::InvalidInput);
Expand Down Expand Up @@ -1253,6 +1266,7 @@ fn window_guard_bounds_automatic_reservation_and_rejects_an_oversized_write() {
256,
SchedulerConfig::default(),
false,
std::sync::Arc::new(()),
)
.is_ok()
);
Expand Down Expand Up @@ -1299,6 +1313,7 @@ fn hybrid_reservation_engine(prefix_cache: bool) -> EngineCore<&'static str> {
..Default::default()
},
prefix_cache,
std::sync::Arc::new(()),
)
.unwrap()
}
Expand Down Expand Up @@ -1468,6 +1483,7 @@ fn hybrid_reservation_supports_distinct_private_state_checkpoint_cadences() {
2048,
SchedulerConfig::default(),
false,
std::sync::Arc::new(()),
)
.unwrap();
let mut model = Model::default();
Expand Down Expand Up @@ -1612,6 +1628,7 @@ fn rounded_reservation_tail_cannot_exceed_a_windows_declared_write_guard() {
256,
SchedulerConfig::default(),
false,
std::sync::Arc::new(()),
)
.unwrap();
let mut model = Model::default();
Expand Down Expand Up @@ -1654,6 +1671,7 @@ fn bounded_reservation_engine(capacity: u32, context: u32) -> EngineCore<&'stati
..Default::default()
},
false,
std::sync::Arc::new(()),
)
.unwrap()
}
Expand Down Expand Up @@ -1849,6 +1867,7 @@ fn stop_overlap_engine() -> EngineCore<&'static str> {
512,
SchedulerConfig::default(),
true,
std::sync::Arc::new(()),
)
.unwrap()
}
Expand Down Expand Up @@ -1988,6 +2007,7 @@ fn retraction_counts_compute_replay_without_duplicating_logical_tokens() {
..Default::default()
},
false,
std::sync::Arc::new(()),
)
.unwrap();
engine.enable_metrics().unwrap();
Expand Down Expand Up @@ -2131,6 +2151,7 @@ fn failure_after_next_enqueue_keeps_transferred_lease_quarantined() {
512,
SchedulerConfig::default(),
false,
std::sync::Arc::new(()),
)
.unwrap();
let mut model = Model::default();
Expand Down Expand Up @@ -2173,6 +2194,7 @@ fn waiting_prefix_owners_are_retracted_before_running_requests_under_pressure()
..Default::default()
},
true,
std::sync::Arc::new(()),
)
.unwrap();
let mut model = Model::default();
Expand Down Expand Up @@ -2245,6 +2267,7 @@ fn failed_prefill_growth_releases_unadmitted_pages_before_retrying_running_work(
schedule_conservativeness: 0.0,
},
true,
std::sync::Arc::new(()),
)
.unwrap();
let mut model = Model::default();
Expand Down Expand Up @@ -2304,6 +2327,7 @@ fn a_displaced_rejected_request_does_not_block_a_higher_priority_request() {
3,
SchedulerConfig::default(),
false,
std::sync::Arc::new(()),
)
.unwrap();
let mut model = Model::default();
Expand All @@ -2328,3 +2352,58 @@ fn a_displaced_rejected_request_does_not_block_a_higher_priority_request() {
assert_eq!(rejected.take_result().unwrap(), [4]);
assert_eq!(engine.cache_free_parent_count(), Some(3));
}

#[test]
fn final_core_drop_retains_actual_backing_until_execution_retirement() {
use std::sync::Arc;
for outcome in ["pending", "failed", "success", "rejected"] {
let backing = Arc::new(vec![73_u8; 64]);
let weak = Arc::downgrade(&backing);
let mut engine = EngineCore::new(
RunnerConfig {
cache_layout: Some(
KvLayout::new(
4,
vec![KvGroupLayout::new(2, 1, KvRetention::FullContext).unwrap()],
)
.unwrap(),
),
..Default::default()
},
1,
32,
SchedulerConfig::default(),
false,
backing,
)
.unwrap();
let mut model = Model::default();
let request = engine.submit(input(vec![1], 1)).unwrap();
engine.prepare_step(&mut model, None).unwrap().unwrap();
if outcome != "pending" {
let rows = owned_rows(vec![vec![2]]);
let result = match outcome {
"failed" => Err(Error::execution("device receipt failed")),
"rejected" => Err(Error::rejected("proved no access")),
_ => Ok(rows.view()),
};
let completed = engine.complete_step_with(&mut model, result, Ok);
assert_eq!(completed.is_ok(), outcome == "success");
}
let uncertain = matches!(outcome, "pending" | "failed");
if uncertain {
assert!(!request.done());
assert_eq!(engine.cache_occupied_block_count(), Some(1));
assert!(engine.close(&mut model).is_err());
assert!(model.released.is_empty());
} else {
engine.close(&mut model).unwrap();
assert_eq!(engine.cache_occupied_block_count(), Some(0));
}
drop(engine);
assert_eq!(weak.upgrade().is_some(), uncertain);
if uncertain {
assert_eq!(weak.upgrade().unwrap().as_slice(), &[73; 64]);
}
}
}
1 change: 1 addition & 0 deletions crates/orbitinfer-core/tests/preparation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ fn engine(prefix: bool) -> EngineCore<Payload> {
..Default::default()
},
prefix,
std::sync::Arc::new(()),
)
.unwrap()
}
Expand Down
1 change: 1 addition & 0 deletions crates/orbitinfer-core/tests/prompt_scores.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ fn engine(chunk: u32) -> EngineCore<()> {
..Default::default()
},
false,
std::sync::Arc::new(()),
)
.unwrap()
}
Expand Down
Loading
Loading