[experiment] No Terminator Option

compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs

@@ -4085,7 +4085,7 @@ impl<'diag, 'tcx> MirBorrowckCtxt<'_, 'diag, 'tcx> {
 
     fn explain_deref_coercion(&mut self, loan: &BorrowData<'tcx>, err: &mut Diag<'_>) {
         let tcx = self.infcx.tcx;
-        if let Some(Terminator { kind: TerminatorKind::Call { call_source, fn_span, .. }, .. }) =
+        if let Terminator { kind: TerminatorKind::Call { call_source, fn_span, .. }, .. } =
             &self.body[loan.reserve_location.block].terminator
             && let Some((method_did, method_args)) = mir::find_self_call(
                 tcx,

compiler/rustc_borrowck/src/diagnostics/find_use.rs

@@ -57,7 +57,7 @@ impl<'a, 'tcx> UseFinder<'a, 'tcx> {
             if is_statement {
                 visitor.visit_statement(&block_data.statements[p.statement_index], p);
             } else {
-                visitor.visit_terminator(block_data.terminator.as_ref().unwrap(), p);
+                visitor.visit_terminator(block_data.terminator(), p);
             }
 
             match visitor.def_use_result {

compiler/rustc_borrowck/src/diagnostics/mod.rs

@@ -561,15 +561,15 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
                     );
                     if !is_terminator {
                         continue;
-                    } else if let Some(Terminator {
+                    } else if let Terminator {
                         kind:
                             TerminatorKind::Call {
                                 func,
                                 call_source: CallSource::OverloadedOperator,
                                 ..
                             },
                         ..
-                    }) = &bbd.terminator
+                    } = &bbd.terminator
                     {
                         if let Some(source) =
                             BorrowedContentSource::from_call(func.ty(self.body, tcx), tcx)
@@ -1082,9 +1082,8 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
 
         debug!("move_spans: target_temp = {:?}", target_temp);
 
-        if let Some(Terminator {
-            kind: TerminatorKind::Call { fn_span, call_source, .. }, ..
-        }) = &self.body[location.block].terminator
+        if let Terminator { kind: TerminatorKind::Call { fn_span, call_source, .. }, .. } =
+            &self.body[location.block].terminator
         {
             let Some((method_did, method_args)) =
                 find_self_call(self.infcx.tcx, self.body, target_temp, location.block)

compiler/rustc_borrowck/src/region_infer/region_context.rs

@@ -823,7 +823,7 @@ impl<'tcx> RegionInferenceContextInner<'tcx> {
         for location in locations {
             if from_location.is_predecessor_of(location, body) {
                 let bb = &body[location.block];
-                if let Some(terminator) = &bb.terminator
+                if let terminator = &bb.terminator
                 // terminator of a loop should be TerminatorKind::FalseUnwind
                 && let TerminatorKind::FalseUnwind { .. } = terminator.kind
                 {

compiler/rustc_middle/src/mir/mod.rs

@@ -1336,7 +1336,7 @@ pub struct BasicBlockData<'tcx> {
     /// exception is that certain passes, such as `simplify_cfg`, swap
     /// out the terminator temporarily with `None` while they continue
     /// to recurse over the set of basic blocks.
-    pub terminator: Option<Terminator<'tcx>>,
+    pub terminator: Terminator<'tcx>,
 
     /// If true, this block lies on an unwind path. This is used
     /// during codegen where distinct kinds of basic blocks may be
@@ -1346,13 +1346,13 @@ pub struct BasicBlockData<'tcx> {
 }
 
 impl<'tcx> BasicBlockData<'tcx> {
-    pub fn new(terminator: Option<Terminator<'tcx>>, is_cleanup: bool) -> BasicBlockData<'tcx> {
+    pub fn new(terminator: Terminator<'tcx>, is_cleanup: bool) -> BasicBlockData<'tcx> {
         BasicBlockData::new_stmts(Vec::new(), terminator, is_cleanup)
     }
 
     pub fn new_stmts(
         statements: Vec<Statement<'tcx>>,
-        terminator: Option<Terminator<'tcx>>,
+        terminator: Terminator<'tcx>,
         is_cleanup: bool,
     ) -> BasicBlockData<'tcx> {
         BasicBlockData {
@@ -1369,12 +1369,12 @@ impl<'tcx> BasicBlockData<'tcx> {
     /// provides a convenient way to reach the terminator.
     #[inline]
     pub fn terminator(&self) -> &Terminator<'tcx> {
-        self.terminator.as_ref().expect("invalid terminator state")
+        &self.terminator
     }
 
     #[inline]
     pub fn terminator_mut(&mut self) -> &mut Terminator<'tcx> {
-        self.terminator.as_mut().expect("invalid terminator state")
+        &mut self.terminator
     }
 
     /// Does the block have no statements and an unreachable terminator?
@@ -1696,7 +1696,7 @@ pub fn find_self_call<'tcx>(
     block: BasicBlock,
 ) -> Option<(DefId, GenericArgs<'tcx>)> {
     debug!("find_self_call(local={:?}): terminator={:?}", local, body[block].terminator);
-    if let Some(Terminator { kind: TerminatorKind::Call { func, args, .. }, .. }) =
+    if let Terminator { kind: TerminatorKind::Call { func, args, .. }, .. } =
         &body[block].terminator
         && let Operand::Constant(ConstOperand { const_, .. }) = func
         && let ty::FnDef(def_id, fn_args) = *const_.ty().kind()

compiler/rustc_middle/src/mir/pretty.rs

@@ -836,7 +836,7 @@ impl<'a, 'tcx> MirWriter<'a, 'tcx> {
 
         // Terminator at the bottom.
         (self.extra_data)(PassWhere::BeforeLocation(current_location), w)?;
-        if data.terminator.is_some() {
+        if true {
             let indented_terminator = format!("{0}{0}{1:?};", INDENT, data.terminator().kind);
             if self.options.include_extra_comments {
                 writeln!(

compiler/rustc_middle/src/mir/terminator.rs

@@ -426,6 +426,27 @@ pub struct Terminator<'tcx> {
     pub kind: TerminatorKind<'tcx>,
 }
 
+impl<'tcx> Terminator<'tcx> {
+    #[inline]
+    pub fn none() -> Terminator<'tcx> {
+        Terminator {
+            source_info: SourceInfo::outermost(DUMMY_SP),
+            kind: TerminatorKind::Goto { target: BasicBlock::MAX, loop_hint_attrs: ThinVec::new() },
+        }
+    }
+
+    pub fn take(&mut self) -> Option<Terminator<'tcx>> {
+        if self.is_none() { None } else { Some(std::mem::replace(self, Self::none())) }
+    }
+
+    pub fn is_none(&self) -> bool {
+        matches!(self.kind, TerminatorKind::Goto { target: BasicBlock::MAX, .. })
+    }
+    pub fn is_some(&self) -> bool {
+        !self.is_none()
+    }
+}
+
 impl<'tcx> Terminator<'tcx> {
     #[inline]
     pub fn successors(&self) -> Successors<'_> {

compiler/rustc_middle/src/mir/traversal.rs

@@ -54,9 +54,7 @@ impl<'a, 'tcx> Iterator for Preorder<'a, 'tcx> {
 
             let data = &self.body[idx];
 
-            if let Some(ref term) = data.terminator {
-                self.worklist.extend(term.successors());
-            }
+            self.worklist.extend(data.terminator.successors());
 
             return Some((idx, data));
         }

compiler/rustc_middle/src/mir/visit.rs

@@ -316,9 +316,7 @@ macro_rules! make_mir_visitor {
                 for debuginfo in after_last_stmt_debuginfos as & $($mutability)? [_] {
                     self.visit_statement_debuginfo(debuginfo, location);
                 }
-                if let Some(terminator) = terminator {
-                    self.visit_terminator(terminator, location);
-                }
+                self.visit_terminator(terminator, location);
             }
 
             fn super_source_scope_data(
@@ -998,9 +996,7 @@ macro_rules! make_mir_visitor {
                 let basic_block =
                     & $($mutability)? basic_blocks!(body, $($mutability, true)?)[location.block];
                 if basic_block.statements.len() == location.statement_index {
-                    if let Some(ref $($mutability)? terminator) = basic_block.terminator {
-                        self.visit_terminator(terminator, location)
-                    }
+                    self.visit_terminator( & $($mutability)? basic_block.terminator, location)
                 } else {
                     let statement = & $($mutability)?
                         basic_block.statements[location.statement_index];

compiler/rustc_mir_build/src/builder/cfg.rs

@@ -21,7 +21,7 @@ impl<'tcx> CFG<'tcx> {
     // it as #[inline(never)] to keep rustc's stack use in check.
     #[inline(never)]
     pub(crate) fn start_new_block(&mut self) -> BasicBlock {
-        self.basic_blocks.push(BasicBlockData::new(None, false))
+        self.basic_blocks.push(BasicBlockData::new(Terminator::none(), false))
     }
 
     pub(crate) fn start_new_cleanup_block(&mut self) -> BasicBlock {
@@ -117,8 +117,8 @@ impl<'tcx> CFG<'tcx> {
             block,
             self.block_data(block)
         );
-        self.block_data_mut(block).terminator = Some(Terminator { source_info, kind });
-        self.block_data_mut(block).terminator.as_mut().unwrap()
+        self.block_data_mut(block).terminator = Terminator { source_info, kind };
+        self.block_data_mut(block).terminator_mut()
     }
 
     /// In the `origin` block, push a `goto -> target` terminator.

compiler/rustc_mir_build/src/builder/custom/mod.rs

@@ -64,7 +64,7 @@ pub(super) fn build_custom_mir<'tcx>(
     };
 
     body.local_decls.push(LocalDecl::new(return_ty, return_ty_span));
-    body.basic_blocks_mut().push(BasicBlockData::new(None, false));
+    body.basic_blocks_mut().push(BasicBlockData::new(Terminator::none(), false));
     body.source_scopes.push(SourceScopeData {
         span,
         parent_scope: None,

compiler/rustc_mir_build/src/builder/custom/parse.rs

@@ -200,7 +200,7 @@ impl<'a, 'tcx> ParseCtxt<'a, 'tcx> {
             StmtKind::Let { pattern, initializer: Some(initializer), .. } => {
                 let (var, ..) = self.parse_var(pattern)?;
                 let data = BasicBlockData::new(
-                    None,
+                    Terminator::none(),
                     parse_by_kind!(self, *initializer, _, "basic block declaration",
                         @variant(mir_basic_block, Normal) => false,
                         @variant(mir_basic_block, Cleanup) => true,
@@ -303,7 +303,7 @@ impl<'a, 'tcx> ParseCtxt<'a, 'tcx> {
             ExprKind::Block { block } => &self.thir[*block],
         );
 
-        let mut data = BasicBlockData::new(None, is_cleanup);
+        let mut data = BasicBlockData::new(Terminator::none(), is_cleanup);
         for stmt_id in &*block.stmts {
             let stmt = self.statement_as_expr(*stmt_id)?;
             let span = self.thir[stmt].span;
@@ -315,10 +315,10 @@ impl<'a, 'tcx> ParseCtxt<'a, 'tcx> {
         let Some(trailing) = block.expr else { return Err(self.expr_error(expr_id, "terminator")) };
         let span = self.thir[trailing].span;
         let terminator = self.parse_terminator(trailing)?;
-        data.terminator = Some(Terminator {
+        data.terminator = Terminator {
             source_info: SourceInfo { span, scope: self.source_scope },
             kind: terminator,
-        });
+        };
 
         Ok(data)
     }

compiler/rustc_mir_transform/src/abort_unwinding_calls.rs

@@ -66,7 +66,7 @@ impl<'tcx> crate::MirPass<'tcx> for AbortUnwindingCalls {
         // This will filter to functions with `extern "C-unwind"` ABIs, for
         // example.
         for block in body.basic_blocks.as_mut() {
-            let Some(terminator) = &mut block.terminator else { continue };
+            let terminator = &mut block.terminator; //else { continue };
             let span = terminator.source_info.span;
 
             // If we see an `UnwindResume` terminator inside a function then:

compiler/rustc_mir_transform/src/add_call_guards.rs

@@ -88,7 +88,7 @@ impl<'tcx> crate::MirPass<'tcx> for AddCallGuards {
         let cur_len = body.basic_blocks.len();
         let mut new_block = |source_info: SourceInfo, is_cleanup: bool, target: BasicBlock| {
             let block = BasicBlockData::new(
-                Some(Terminator { source_info, kind: TerminatorKind::goto(target) }),
+                Terminator { source_info, kind: TerminatorKind::goto(target) },
                 is_cleanup,
             );
             let idx = cur_len + new_blocks.len();

compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs

@@ -94,7 +94,7 @@ fn add_move_for_packed_drop<'tcx>(
 
     let storage_dead_block = patch.new_block(BasicBlockData::new_stmts(
         vec![Statement::new(source_info, StatementKind::StorageDead(temp))],
-        Some(Terminator { source_info, kind: TerminatorKind::goto(target) }),
+        Terminator { source_info, kind: TerminatorKind::goto(target) },
         is_cleanup,
     ));
 

compiler/rustc_mir_transform/src/check_enums.rs

@@ -60,10 +60,8 @@ impl<'tcx> crate::MirPass<'tcx> for CheckEnums {
                                 source_info.span,
                                 "cannot build enum discriminant from zero-sized type",
                             );
-                            basic_blocks[block].terminator = Some(Terminator {
-                                source_info,
-                                kind: TerminatorKind::goto(new_block),
-                            });
+                            basic_blocks[block].terminator =
+                                Terminator { source_info, kind: TerminatorKind::goto(new_block) };
                         }
                         EnumCheckType::Direct { source_op, discr, op_size, valid_discrs } => {
                             insert_direct_enum_check(
@@ -246,7 +244,7 @@ fn split_block(
     // Drain every statement after this one and move the current terminator to a new basic block.
     let new_block = BasicBlockData::new_stmts(
         block_data.statements.split_off(location.statement_index),
-        block_data.terminator.take(),
+        block_data.terminator.take().unwrap(),
         block_data.is_cleanup,
     );
 
@@ -348,7 +346,7 @@ fn insert_direct_enum_check<'tcx>(
     new_block: BasicBlock,
 ) {
     // Insert a new target block that is branched to in case of an invalid discriminant.
-    let invalid_discr_block_data = BasicBlockData::new(None, false);
+    let invalid_discr_block_data = BasicBlockData::new(Terminator::none(), false);
     let invalid_discr_block = basic_blocks.push(invalid_discr_block_data);
     let block_data = &mut basic_blocks[current_block];
     let discr_place = insert_discr_cast_to_u128(
@@ -382,7 +380,7 @@ fn insert_direct_enum_check<'tcx>(
         .push(Statement::new(source_info, StatementKind::Assign(Box::new((discr_masked, rvalue)))));
 
     // Branch based on the discriminant value.
-    block_data.terminator = Some(Terminator {
+    block_data.terminator = Terminator {
         source_info,
         kind: TerminatorKind::SwitchInt {
             discr: Operand::Copy(discr_masked),
@@ -393,10 +391,10 @@ fn insert_direct_enum_check<'tcx>(
                 invalid_discr_block,
             ),
         },
-    });
+    };
 
     // Abort in case of an invalid enum discriminant.
-    basic_blocks[invalid_discr_block].terminator = Some(Terminator {
+    basic_blocks[invalid_discr_block].terminator = Terminator {
         source_info,
         kind: TerminatorKind::Assert {
             cond: Operand::Constant(Box::new(ConstOperand {
@@ -412,7 +410,7 @@ fn insert_direct_enum_check<'tcx>(
             // make a failing UB check turn into much worse UB when we start unwinding.
             unwind: UnwindAction::Unreachable,
         },
-    });
+    };
 }
 
 fn insert_uninhabited_enum_check<'tcx>(
@@ -439,7 +437,7 @@ fn insert_uninhabited_enum_check<'tcx>(
         ))),
     ));
 
-    block_data.terminator = Some(Terminator {
+    block_data.terminator = Terminator {
         source_info,
         kind: TerminatorKind::Assert {
             cond: Operand::Copy(is_ok),
@@ -457,7 +455,7 @@ fn insert_uninhabited_enum_check<'tcx>(
             // make a failing UB check turn into much worse UB when we start unwinding.
             unwind: UnwindAction::Unreachable,
         },
-    });
+    };
 }
 
 fn insert_niche_check<'tcx>(
@@ -522,7 +520,7 @@ fn insert_niche_check<'tcx>(
         ))),
     ));
 
-    block_data.terminator = Some(Terminator {
+    block_data.terminator = Terminator {
         source_info,
         kind: TerminatorKind::Assert {
             cond: Operand::Copy(is_ok),
@@ -534,5 +532,5 @@ fn insert_niche_check<'tcx>(
             // make a failing UB check turn into much worse UB when we start unwinding.
             unwind: UnwindAction::Unreachable,
         },
-    });
+    };
 }

compiler/rustc_mir_transform/src/check_pointers.rs

@@ -103,7 +103,7 @@ pub(crate) fn check_pointers<'tcx, F>(
                     &mut block_data.statements,
                     source_info,
                 );
-                block_data.terminator = Some(Terminator {
+                block_data.terminator = Terminator {
                     source_info,
                     kind: TerminatorKind::Assert {
                         cond: pointer_check.cond,
@@ -116,7 +116,7 @@ pub(crate) fn check_pointers<'tcx, F>(
                         // worse UB when we start unwinding.
                         unwind: UnwindAction::Unreachable,
                     },
-                });
+                };
             }
         }
     }
@@ -237,7 +237,7 @@ fn split_block(
     // Drain every statement after this one and move the current terminator to a new basic block.
     let new_block = BasicBlockData::new_stmts(
         block_data.statements.split_off(location.statement_index),
-        block_data.terminator.take(),
+        block_data.terminator.take().unwrap(), // fixme I don't know whether this unwrap is correct
         block_data.is_cleanup,
     );