From 725149c423aa200268d28bb75906d0af6a98683a Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 4 Nov 2026 22:19:08 +0100 Subject: [PATCH 1/5] join 32-bit and 64-bit virtual function elimination tests --- .../src/debuginfo/metadata.rs | 2 +- .../virtual-function-elimination-32bit.rs | 35 ------------------- .../virtual-function-elimination.rs | 23 +++++++----- 3 files changed, 16 insertions(+), 44 deletions(-) delete mode 100644 tests/codegen-llvm/virtual-function-elimination-32bit.rs diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 752cbc67d5637..4b8bf2662a300 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -1787,7 +1787,7 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref); let typeid = cx.create_metadata(trait_ref_typeid.as_bytes()); - let type_ = [llvm::LLVMValueAsMetadata(cx.const_usize(0)), typeid]; + let type_ = [llvm::LLVMValueAsMetadata(cx.const_u64(0)), typeid]; cx.global_add_metadata_node(vtable, llvm::MD_type, &type_); let vcall_visibility = [llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64))]; diff --git a/tests/codegen-llvm/virtual-function-elimination-32bit.rs b/tests/codegen-llvm/virtual-function-elimination-32bit.rs deleted file mode 100644 index c9919cecccf9b..0000000000000 --- a/tests/codegen-llvm/virtual-function-elimination-32bit.rs +++ /dev/null @@ -1,35 +0,0 @@ -//@ compile-flags: -Zvirtual-function-elimination -Clto -Copt-level=3 -Csymbol-mangling-version=v0 -//@ ignore-64bit - -// CHECK: @vtable.0 = {{.*}}, !type ![[TYPE0:[0-9]+]], !vcall_visibility ![[VCALL_VIS0:[0-9]+]] - -#![crate_type = "lib"] - -trait T { - // CHECK-LABEL: ; ::used - fn used(&self) -> i32 { - 1 - } - // CHECK-LABEL-NOT: {{.*}}::unused - fn unused(&self) -> i32 { - 2 - } -} - -#[derive(Copy, Clone)] -struct S; - -impl T for S {} - -fn taking_t(t: &dyn T) -> i32 { - // CHECK: @llvm.type.checked.load({{.*}}, i32 12, metadata !"[[MANGLED_TYPE0:[0-9a-zA-Z_]+]]") - t.used() -} - -pub fn main() { - let s = S; - taking_t(&s); -} - -// CHECK: ![[TYPE0]] = !{i32 0, !"[[MANGLED_TYPE0]]"} -// CHECK: ![[VCALL_VIS0]] = !{i64 2} diff --git a/tests/codegen-llvm/virtual-function-elimination.rs b/tests/codegen-llvm/virtual-function-elimination.rs index 26604478c11a2..90331a4c59f3e 100644 --- a/tests/codegen-llvm/virtual-function-elimination.rs +++ b/tests/codegen-llvm/virtual-function-elimination.rs @@ -1,8 +1,7 @@ //@ compile-flags: -Zvirtual-function-elimination -Clto -Copt-level=3 -Csymbol-mangling-version=v0 -//@ ignore-32bit // CHECK: @vtable.0 = {{.*}}, !type ![[TYPE0:[0-9]+]], !vcall_visibility ![[VCALL_VIS0:[0-9]+]] -// CHECK: @vtable.1 = {{.*}}, !type ![[TYPE1:[0-9]+]], !vcall_visibility ![[VCALL_VIS0:[0-9]+]] +// CHECK: @vtable.1 = {{.*}}, !type ![[TYPE1:[0-9]+]], !vcall_visibility ![[VCALL_VIS0]] // CHECK: @vtable.2 = {{.*}}, !type ![[TYPE2:[0-9]+]], !vcall_visibility ![[VCALL_VIS2:[0-9]+]] #![crate_type = "lib"] @@ -61,25 +60,33 @@ impl V for S { } } +// CHECK-LABEL: taking_t +#[unsafe(no_mangle)] fn taking_t(t: &dyn T) -> i32 { - // CHECK: @llvm.type.checked.load({{.*}}, i32 24, metadata !"[[MANGLED_TYPE0:[0-9a-zA-Z_]+]]") + // CHECK: @llvm.type.checked.load({{.*}}, i32 {{12|24}}, metadata !"[[MANGLED_TYPE0:[0-9a-zA-Z_]+]]") t.used() } +// CHECK-LABEL: taking_rc_t +#[unsafe(no_mangle)] fn taking_rc_t(t: Rc) -> i32 { - // CHECK: @llvm.type.checked.load({{.*}}, i32 40, metadata !"[[MANGLED_TYPE0:[0-9a-zA-Z_]+]]") + // CHECK: @llvm.type.checked.load({{.*}}, i32 {{20|40}}, metadata !"[[MANGLED_TYPE0:[0-9a-zA-Z_]+]]") t.by_rc() } +// CHECK-LABEL: taking_u +#[unsafe(no_mangle)] fn taking_u(u: &dyn U) -> i32 { - // CHECK: @llvm.type.checked.load({{.*}}, i32 64, metadata !"[[MANGLED_TYPE1:[0-9a-zA-Z_]+]]") - // CHECK: @llvm.type.checked.load({{.*}}, i32 24, metadata !"[[MANGLED_TYPE1:[0-9a-zA-Z_]+]]") - // CHECK: @llvm.type.checked.load({{.*}}, i32 32, metadata !"[[MANGLED_TYPE1:[0-9a-zA-Z_]+]]") + // CHECK: @llvm.type.checked.load({{.*}}, i32 {{32|64}}, metadata !"[[MANGLED_TYPE1:[0-9a-zA-Z_]+]]") + // CHECK: @llvm.type.checked.load({{.*}}, i32 {{12|24}}, metadata !"[[MANGLED_TYPE1:[0-9a-zA-Z_]+]]") + // CHECK: @llvm.type.checked.load({{.*}}, i32 {{16|32}}, metadata !"[[MANGLED_TYPE1:[0-9a-zA-Z_]+]]") u.subtrait_used() + u.used() + u.used_through_sub_trait() } +// CHECK-LABEL: taking_v +#[unsafe(no_mangle)] pub fn taking_v(v: &dyn V) -> i32 { - // CHECK: @llvm.type.checked.load({{.*}}, i32 24, metadata !"NtC[[CRATE_IDENT:[a-zA-Z0-9]{12}]]_28virtual_function_elimination1V") + // CHECK: @llvm.type.checked.load({{.*}}, i32 {{12|24}}, metadata !"NtC[[CRATE_IDENT:[a-zA-Z0-9]{12}]]_28virtual_function_elimination1V") v.public_function() } From db38a7a5e4c53ce9def9333a778beaafdacbf271 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 4 Nov 2026 17:41:19 +0100 Subject: [PATCH 2/5] use `trait_ref`'s def_id In `with_self_ty` the def_id that is used is just `self.def_id`, so this is always the same DefId --- compiler/rustc_codegen_llvm/src/context.rs | 3 +-- .../rustc_codegen_llvm/src/debuginfo/metadata.rs | 5 +---- compiler/rustc_codegen_ssa/src/meth.rs | 12 ++++++------ compiler/rustc_codegen_ssa/src/traits/misc.rs | 1 - 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 4291408212358..1cd2883565d56 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -941,11 +941,10 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn apply_vcall_visibility_metadata( &self, - ty: Ty<'tcx>, poly_trait_ref: Option>, vtable: &'ll Value, ) { - apply_vcall_visibility_metadata(self, ty, poly_trait_ref, vtable); + apply_vcall_visibility_metadata(self, poly_trait_ref, vtable); } fn get_fn(&self, instance: Instance<'tcx>) -> &'ll Value { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 4b8bf2662a300..1b3d209eb07d4 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -1735,7 +1735,6 @@ fn find_vtable_behind_cast<'ll>(vtable: &'ll Value) -> &'ll Value { pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - ty: Ty<'tcx>, trait_ref: Option>, vtable: &'ll Value, ) { @@ -1755,9 +1754,7 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( // Unwrap potential addrspacecast let vtable = find_vtable_behind_cast(vtable); - let trait_ref_self = trait_ref.with_self_ty(cx.tcx, ty); - let trait_def_id = trait_ref_self.def_id; - let trait_vis = cx.tcx.visibility(trait_def_id); + let trait_vis = cx.tcx.visibility(trait_ref.def_id); let cgus = cx.sess().codegen_units().as_usize(); let single_cgu = cgus == 1; diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs index 5b945fc3c2763..97553f79ae238 100644 --- a/compiler/rustc_codegen_ssa/src/meth.rs +++ b/compiler/rustc_codegen_ssa/src/meth.rs @@ -143,22 +143,22 @@ fn dyn_trait_in_self<'tcx>( #[instrument(level = "debug", skip(cx))] pub(crate) fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>( cx: &Cx, - ty: Ty<'tcx>, + self_ty: Ty<'tcx>, trait_ref: Option>, ) -> Cx::Value { let tcx = cx.tcx(); // Check the cache. - if let Some(&val) = cx.vtables().borrow().get(&(ty, trait_ref)) { + if let Some(&val) = cx.vtables().borrow().get(&(self_ty, trait_ref)) { return val; } - let vtable_alloc_id = tcx.vtable_allocation((ty, trait_ref)); + let vtable_alloc_id = tcx.vtable_allocation((self_ty, trait_ref)); let vtable_allocation = tcx.global_alloc(vtable_alloc_id).unwrap_memory(); let vtable = cx.static_addr_of(vtable_allocation, Some("vtable")); - cx.apply_vcall_visibility_metadata(ty, trait_ref, vtable); - cx.create_vtable_debuginfo(ty, trait_ref, vtable); - cx.vtables().borrow_mut().insert((ty, trait_ref), vtable); + cx.apply_vcall_visibility_metadata(trait_ref, vtable); + cx.create_vtable_debuginfo(self_ty, trait_ref, vtable); + cx.vtables().borrow_mut().insert((self_ty, trait_ref), vtable); vtable } diff --git a/compiler/rustc_codegen_ssa/src/traits/misc.rs b/compiler/rustc_codegen_ssa/src/traits/misc.rs index add7128a2974b..b2ee95f69c508 100644 --- a/compiler/rustc_codegen_ssa/src/traits/misc.rs +++ b/compiler/rustc_codegen_ssa/src/traits/misc.rs @@ -13,7 +13,6 @@ pub trait MiscCodegenMethods<'tcx>: BackendTypes { ) -> &RefCell, Option>), Self::Value>>; fn apply_vcall_visibility_metadata( &self, - _ty: Ty<'tcx>, _poly_trait_ref: Option>, _vtable: Self::Value, ) { From c91e5f26f97c677cc742469222312a9b899f2fa9 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 4 Nov 2026 21:19:43 +0100 Subject: [PATCH 3/5] document and refactor `VCallVisibility` --- .../src/debuginfo/metadata.rs | 77 +++++++++++-------- 1 file changed, 45 insertions(+), 32 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 1b3d209eb07d4..96d3e56687af4 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -1733,6 +1733,49 @@ fn find_vtable_behind_cast<'ll>(vtable: &'ll Value) -> &'ll Value { vtable } +/// From where can virtual calls using this vtable be made? +/// +/// See also . +#[derive(PartialEq, Eq, PartialOrd, Ord)] +enum VCallVisibility { + /// Virtual function calls using this vtable could be made from external code. + Public = 0, + /// All virtual function calls which might use this vtable are in the current LTO unit, + /// meaning they will be in the current module once LTO linking has been performed. + LinkageUnit = 1, + /// All virtual function calls which might use this vtable are in the current module. + TranslationUnit = 2, +} + +impl VCallVisibility { + fn for_def_id<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, def_id: DefId) -> Self { + let lto = cx.sess().lto(); + let trait_vis = cx.tcx.visibility(def_id); + let cgus = cx.sess().codegen_units().as_usize(); + let single_cgu = cgus == 1; + + // Because LLVM requires full LTO for the virtual function elimination optimization to + // apply, only the `Lto::Fat` cases are relevant currently. + match (lto, trait_vis, single_cgu) { + // If there is not LTO and the visibility in public, we have to assume that the vtable + // can be seen from anywhere. With multiple CGUs, the vtable is quasi-public. + (Lto::No | Lto::ThinLocal, Visibility::Public, _) + | (Lto::No, Visibility::Restricted(_), false) => VCallVisibility::Public, + // With LTO and a quasi-public visibility, the usages of the functions of the vtable are + // all known by the `LinkageUnit`. + // FIXME: LLVM only supports this optimization for `Lto::Fat` currently. Once it also + // supports `Lto::Thin` the `VCallVisibility` may have to be adjusted for those. + (Lto::Fat | Lto::Thin, Visibility::Public, _) + | (Lto::ThinLocal | Lto::Thin | Lto::Fat, Visibility::Restricted(_), false) => { + VCallVisibility::LinkageUnit + } + // If there is only one CGU, private vtables can only be seen by that CGU/translation + // unit and therefore we know of all usages of functions in the vtable. + (_, Visibility::Restricted(_), true) => VCallVisibility::TranslationUnit, + } + } +} + pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, trait_ref: Option>, @@ -1744,42 +1787,12 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( return; } - enum VCallVisibility { - Public = 0, - LinkageUnit = 1, - TranslationUnit = 2, - } - let Some(trait_ref) = trait_ref else { return }; // Unwrap potential addrspacecast let vtable = find_vtable_behind_cast(vtable); - let trait_vis = cx.tcx.visibility(trait_ref.def_id); - - let cgus = cx.sess().codegen_units().as_usize(); - let single_cgu = cgus == 1; - - let lto = cx.sess().lto(); - - // Since LLVM requires full LTO for the virtual function elimination optimization to apply, - // only the `Lto::Fat` cases are relevant currently. - let vcall_visibility = match (lto, trait_vis, single_cgu) { - // If there is not LTO and the visibility in public, we have to assume that the vtable can - // be seen from anywhere. With multiple CGUs, the vtable is quasi-public. - (Lto::No | Lto::ThinLocal, Visibility::Public, _) - | (Lto::No, Visibility::Restricted(_), false) => VCallVisibility::Public, - // With LTO and a quasi-public visibility, the usages of the functions of the vtable are - // all known by the `LinkageUnit`. - // FIXME: LLVM only supports this optimization for `Lto::Fat` currently. Once it also - // supports `Lto::Thin` the `VCallVisibility` may have to be adjusted for those. - (Lto::Fat | Lto::Thin, Visibility::Public, _) - | (Lto::ThinLocal | Lto::Thin | Lto::Fat, Visibility::Restricted(_), false) => { - VCallVisibility::LinkageUnit - } - // If there is only one CGU, private vtables can only be seen by that CGU/translation unit - // and therefore we know of all usages of functions in the vtable. - (_, Visibility::Restricted(_), true) => VCallVisibility::TranslationUnit, - }; + + let vcall_visibility = VCallVisibility::for_def_id(cx, trait_ref.def_id); let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref); let typeid = cx.create_metadata(trait_ref_typeid.as_bytes()); From 464affb7381fe98f089411b9e69a22ed2801bbb6 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 4 Nov 2026 21:22:44 +0100 Subject: [PATCH 4/5] support a slice of (super)traits in `apply_vcall_visibility_metadata` For now it gets at most one trait, the next step is to (recursively) provide supertraits too --- compiler/rustc_codegen_llvm/src/context.rs | 4 +-- .../src/debuginfo/metadata.rs | 31 ++++++++++++++----- compiler/rustc_codegen_ssa/src/meth.rs | 2 +- compiler/rustc_codegen_ssa/src/traits/misc.rs | 2 +- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 1cd2883565d56..888d4ab374d0d 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -941,10 +941,10 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn apply_vcall_visibility_metadata( &self, - poly_trait_ref: Option>, + trait_hierarchy: &[ty::ExistentialTraitRef<'tcx>], vtable: &'ll Value, ) { - apply_vcall_visibility_metadata(self, poly_trait_ref, vtable); + apply_vcall_visibility_metadata(self, trait_hierarchy, vtable); } fn get_fn(&self, instance: Instance<'tcx>) -> &'ll Value { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 96d3e56687af4..e527c3b652310 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -1776,9 +1776,13 @@ impl VCallVisibility { } } +/// Apply the virtual call visibility metadata to the vtable. +/// +/// Adds a !type annotation for every trait in the trait hierarchy, and a !vcall_visibility +/// annotation based on the most visible trait in the hierarchy. pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - trait_ref: Option>, + trait_hierarchy: &[ExistentialTraitRef<'tcx>], vtable: &'ll Value, ) { // FIXME(flip1995): The virtual function elimination optimization only works with full LTO in @@ -1787,19 +1791,30 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( return; } - let Some(trait_ref) = trait_ref else { return }; + let Some(vcall_visibility) = trait_hierarchy + .iter() + .map(|trait_ref| VCallVisibility::for_def_id(cx, trait_ref.def_id)) + .min() + else { + return; + }; - // Unwrap potential addrspacecast + // Unwrap a potential addrspacecast. let vtable = find_vtable_behind_cast(vtable); - let vcall_visibility = VCallVisibility::for_def_id(cx, trait_ref.def_id); + // The start of a Rust vtable is at offset 0 of a `&dyn T`'s address. + let vtable_offset = 0; - let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref); - let typeid = cx.create_metadata(trait_ref_typeid.as_bytes()); + // Add `!type !{i64 0, !"typeid"}` to the vtable for each trait in the hierarchy. + for trait_ref in trait_hierarchy { + let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, *trait_ref); + let typeid = cx.create_metadata(trait_ref_typeid.as_bytes()); - let type_ = [llvm::LLVMValueAsMetadata(cx.const_u64(0)), typeid]; - cx.global_add_metadata_node(vtable, llvm::MD_type, &type_); + let type_ = [llvm::LLVMValueAsMetadata(cx.const_u64(vtable_offset)), typeid]; + cx.global_add_metadata_node(vtable, llvm::MD_type, &type_); + } + // Add `!vcall_visibility !{i64 vis}` to the vtable. let vcall_visibility = [llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64))]; cx.global_set_metadata_node(vtable, llvm::MD_vcall_visibility, &vcall_visibility); } diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs index 97553f79ae238..3705a9f79bf56 100644 --- a/compiler/rustc_codegen_ssa/src/meth.rs +++ b/compiler/rustc_codegen_ssa/src/meth.rs @@ -157,7 +157,7 @@ pub(crate) fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>( let vtable_allocation = tcx.global_alloc(vtable_alloc_id).unwrap_memory(); let vtable = cx.static_addr_of(vtable_allocation, Some("vtable")); - cx.apply_vcall_visibility_metadata(trait_ref, vtable); + cx.apply_vcall_visibility_metadata(trait_ref.as_slice(), vtable); cx.create_vtable_debuginfo(self_ty, trait_ref, vtable); cx.vtables().borrow_mut().insert((self_ty, trait_ref), vtable); vtable diff --git a/compiler/rustc_codegen_ssa/src/traits/misc.rs b/compiler/rustc_codegen_ssa/src/traits/misc.rs index b2ee95f69c508..444ed78557ddb 100644 --- a/compiler/rustc_codegen_ssa/src/traits/misc.rs +++ b/compiler/rustc_codegen_ssa/src/traits/misc.rs @@ -13,7 +13,7 @@ pub trait MiscCodegenMethods<'tcx>: BackendTypes { ) -> &RefCell, Option>), Self::Value>>; fn apply_vcall_visibility_metadata( &self, - _poly_trait_ref: Option>, + _trait_hierarchy: &[ty::ExistentialTraitRef<'tcx>], _vtable: Self::Value, ) { } From c638b49dde1ebf924dcb902df8f9ac1b02dc5385 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 4 Nov 2026 21:45:02 +0100 Subject: [PATCH 5/5] add !type annotations for each (super)trait that is a prefix of a given vtable --- compiler/rustc_codegen_ssa/src/meth.rs | 13 ++- .../src/traits/vtable.rs | 34 +++++++ tests/codegen-llvm/vfe-annotate-supertrait.rs | 95 +++++++++++++++++++ tests/codegen-llvm/vfe-upcast.rs | 54 +++++++++++ .../virtual-function-elimination.rs | 8 +- tests/ui/codegen/vfe-drop-meta-sized.rs | 58 +++++++++++ 6 files changed, 258 insertions(+), 4 deletions(-) create mode 100644 tests/codegen-llvm/vfe-annotate-supertrait.rs create mode 100644 tests/codegen-llvm/vfe-upcast.rs create mode 100644 tests/ui/codegen/vfe-drop-meta-sized.rs diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs index 3705a9f79bf56..ea1a7d9724d07 100644 --- a/compiler/rustc_codegen_ssa/src/meth.rs +++ b/compiler/rustc_codegen_ssa/src/meth.rs @@ -3,6 +3,7 @@ use rustc_session::config::Lto; use rustc_span::bug; use rustc_symbol_mangling::typeid_for_trait_ref; use rustc_target::callconv::FnAbi; +use rustc_trait_selection::traits::vtable; use tracing::{debug, instrument}; use crate::traits::*; @@ -157,8 +158,18 @@ pub(crate) fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>( let vtable_allocation = tcx.global_alloc(vtable_alloc_id).unwrap_memory(); let vtable = cx.static_addr_of(vtable_allocation, Some("vtable")); - cx.apply_vcall_visibility_metadata(trait_ref.as_slice(), vtable); + if let Some(trait_ref) = trait_ref + && cx.sess().opts.unstable_opts.virtual_function_elimination + && cx.sess().lto() == Lto::Fat + { + let key = trait_ref.with_self_ty(tcx, self_ty); + let trait_hierarchy = vtable::vtable_prefix_traits(tcx, key); + + cx.apply_vcall_visibility_metadata(&trait_hierarchy, vtable); + } + cx.create_vtable_debuginfo(self_ty, trait_ref, vtable); cx.vtables().borrow_mut().insert((self_ty, trait_ref), vtable); + vtable } diff --git a/compiler/rustc_trait_selection/src/traits/vtable.rs b/compiler/rustc_trait_selection/src/traits/vtable.rs index 6dd998791d2dd..845c535de9dda 100644 --- a/compiler/rustc_trait_selection/src/traits/vtable.rs +++ b/compiler/rustc_trait_selection/src/traits/vtable.rs @@ -16,7 +16,9 @@ use crate::traits::is_vtable_safe_method; #[derive(Clone, Debug)] pub enum VtblSegment<'tcx> { + /// The segment containing Drop, Size and Align. MetadataDSA, + /// A segment with the methods of a particular trait. TraitOwnEntries { trait_ref: ty::TraitRef<'tcx>, emit_vptr: bool }, } @@ -453,6 +455,38 @@ pub(crate) fn supertrait_vtable_slot<'tcx>( }) } +/// Returns the traits that are prefixes of this vtable, such that upcasting to these traits +/// reuses the vtable pointer unchanged. +/// +/// For a `trait Foo: A + B`, the methods for A are stored as a prefix to the vtable for Foo. +/// If B has no methods, it is also part of this prefix (although it takes up no space). +/// +/// This function returns the full set of these traits (A might have its own supertrait, etc.). +/// The input trait is the last element of the output vector. +pub fn vtable_prefix_traits<'tcx>( + tcx: TyCtxt<'tcx>, + key: ty::TraitRef<'tcx>, +) -> Vec> { + debug_assert!(!key.has_non_region_infer() && !key.has_non_region_param()); + + let mut entries = vec![]; + + let vtable_segment_callback = |segment| -> ControlFlow<()> { + if let VtblSegment::TraitOwnEntries { trait_ref, emit_vptr: false } = segment { + entries.push(ty::ExistentialTraitRef::erase_self_ty(tcx, trait_ref)); + } + + ControlFlow::Continue(()) + }; + + let _ = prepare_vtable_segments(tcx, key, vtable_segment_callback); + + // The input trait is the last element in the output vector. + debug_assert_eq!(entries.last(), Some(&ty::ExistentialTraitRef::erase_self_ty(tcx, key))); + + entries +} + pub(super) fn provide(providers: &mut Providers) { *providers = Providers { own_existential_vtable_entries, diff --git a/tests/codegen-llvm/vfe-annotate-supertrait.rs b/tests/codegen-llvm/vfe-annotate-supertrait.rs new file mode 100644 index 0000000000000..a1777c19bc864 --- /dev/null +++ b/tests/codegen-llvm/vfe-annotate-supertrait.rs @@ -0,0 +1,95 @@ +//@ compile-flags: -Zvirtual-function-elimination -Clto=fat -Copt-level=3 -Zverify-llvm-ir +//@ no-prefer-dynamic +//@ min-llvm-version: 23 +#![crate_type = "staticlib"] + +// Check that vtables include a !type annotation for supertraits whose vtable are a prefix. +// +// What is still missing is that for any supertrait whose vtable is not a prefix, its vtable does +// not get any !type annotation. That means we provide incomplete information to VFE and it may +// incorrectly replace a virtual call with a direct call to the only callee VFE knows about. + +use std::hint::black_box; + +// FIXME: C_VTABLE should have a !type annotation for C, but it currently has none. +// CHECK: @vtable.{{[0-9]+}} = {{.*}}3Foo3foo{{.*}} !type ![[META_SIZED:[0-9]+]], !type ![[A:[0-9]+]], !type ![[B:[0-9]+]], !type ![[FOO:[0-9]+]], !guid +// CHECK: @[[C_VTABLE:alloc_[0-9a-f]+]] = {{.*}}1C1c }>, align {{[0-9]+}}, !guid +// CHECK: @vtable.{{[0-9]+}} = {{.*}}1C1c, ptr @[[C_VTABLE]], ptr {{.*}}3Bar3bar{{.*}} !type ![[META_SIZED]], !type ![[A]], !type ![[BAR:[0-9]+]], !guid + +// CHECK-DAG: ![[META_SIZED]] = !{i64 0, !"{{.*}}9MetaSized"} +// CHECK-DAG: ![[A]] = !{i64 0, !"{{.*}}1A"} +// CHECK-DAG: ![[B]] = !{i64 0, !"{{.*}}1B"} +// CHECK-DAG: ![[FOO]] = !{i64 0, !"{{.*}}3Foo"} +// CHECK-DAG: ![[BAR]] = !{i64 0, !"{{.*}}3Bar"} + +pub trait A { + fn a(&self) -> i32; +} + +pub trait B {} + +pub trait C { + fn c(&self) -> i32; +} + +/// A trait with two supertraits: +/// +/// - A is a prefix of Foo, because it is the first supertrait in the list +/// - B is a prefix of Foo, because it has no methods +pub trait Foo: A + B { + fn foo(&self) -> i32; +} + +/// A trait with two supertraits: +/// +/// - A is a prefix of Bar, because it is the first supertrait in the list +/// - C is a NOT a prefix of Bar, because it has methods +pub trait Bar: A + C { + fn bar(&self) -> i32; +} + +// The remainder is uninteresting, and only here so that the optimizer can't optimize out the +// interesting bits. + +pub struct S; + +impl A for S { + fn a(&self) -> i32 { + black_box(1) + } +} + +impl B for S {} + +impl C for S { + fn c(&self) -> i32 { + black_box(2) + } +} + +impl Foo for S { + fn foo(&self) -> i32 { + black_box(3) + } +} + +impl Bar for S { + fn bar(&self) -> i32 { + black_box(4) + } +} + +#[unsafe(no_mangle)] +pub fn use_foo(x: &dyn Foo) -> i32 { + x.foo() + x.a() +} + +#[unsafe(no_mangle)] +pub fn use_bar(x: &dyn Bar) -> i32 { + x.bar() + x.a() + x.c() +} + +#[unsafe(no_mangle)] +pub fn use_s() -> i32 { + use_foo(black_box(&S)) + use_bar(black_box(&S)) +} diff --git a/tests/codegen-llvm/vfe-upcast.rs b/tests/codegen-llvm/vfe-upcast.rs new file mode 100644 index 0000000000000..4216dae92b622 --- /dev/null +++ b/tests/codegen-llvm/vfe-upcast.rs @@ -0,0 +1,54 @@ +//@ compile-flags: -Zvirtual-function-elimination -Clto=fat -Copt-level=3 -Zverify-llvm-ir +//@ no-prefer-dynamic +#![crate_type = "staticlib"] + +// Check that a vtable also gets a !type annotation for any supertraits it has whose vtable is a +// prefix of the base trait's. For `trait Foo: A + B` this is true for A because it is the first, +// and also for B if it has no trait methods. +// +// Without this annotation, -Zvirtual-function-elimination may incorrectly remove a vtable if it +// only used by the supertrait. + +use std::hint::black_box; + +pub trait Super { + fn s(&self) -> i32; +} + +pub trait Sub: Super {} + +pub struct A; +pub struct B; + +impl Super for A { + fn s(&self) -> i32 { + black_box(1) + } +} + +impl Sub for A {} + +impl Super for B { + fn s(&self) -> i32 { + black_box(2) + } +} + +// CHECK-LABEL: @call_super +#[unsafe(no_mangle)] +pub fn call_super(x: &dyn Super) -> i32 { + // This call should not be devirtualized, x might be either A or B. + // CHECK: call noundef i32 %{{.*}}(ptr + x.s() +} + +#[unsafe(no_mangle)] +pub fn call_a() -> i32 { + let a: &dyn Sub = black_box(&A); + call_super(a) +} + +#[unsafe(no_mangle)] +pub fn call_b() -> i32 { + call_super(black_box(&B)) +} diff --git a/tests/codegen-llvm/virtual-function-elimination.rs b/tests/codegen-llvm/virtual-function-elimination.rs index 90331a4c59f3e..327b3226a77d8 100644 --- a/tests/codegen-llvm/virtual-function-elimination.rs +++ b/tests/codegen-llvm/virtual-function-elimination.rs @@ -2,7 +2,7 @@ // CHECK: @vtable.0 = {{.*}}, !type ![[TYPE0:[0-9]+]], !vcall_visibility ![[VCALL_VIS0:[0-9]+]] // CHECK: @vtable.1 = {{.*}}, !type ![[TYPE1:[0-9]+]], !vcall_visibility ![[VCALL_VIS0]] -// CHECK: @vtable.2 = {{.*}}, !type ![[TYPE2:[0-9]+]], !vcall_visibility ![[VCALL_VIS2:[0-9]+]] +// CHECK: @vtable.2 = {{.*}}, !type ![[TYPE2:[0-9]+]], !vcall_visibility ![[VCALL_VIS0]] #![crate_type = "lib"] @@ -98,8 +98,10 @@ pub fn main() { taking_v(&s); } +// MetaSized is an implicit supertrait of all dyn-compatible traits. +// CHECK: !0 = !{i64 0, !"[[MANGLED_CORE_MARKER:[0-9a-zA-Z_]+]]MetaSized"} +// // CHECK: ![[TYPE0]] = !{i64 0, !"[[MANGLED_TYPE0]]"} -// CHECK: ![[VCALL_VIS0]] = !{i64 2} +// CHECK: ![[VCALL_VIS0]] = !{i64 1} // CHECK: ![[TYPE1]] = !{i64 0, !"[[MANGLED_TYPE1]]"} // CHECK: ![[TYPE2]] = !{i64 0, !"NtC[[CRATE_IDENT]]_28virtual_function_elimination1V"} -// CHECK: ![[VCALL_VIS2]] = !{i64 1} diff --git a/tests/ui/codegen/vfe-drop-meta-sized.rs b/tests/ui/codegen/vfe-drop-meta-sized.rs new file mode 100644 index 0000000000000..540c41ae8b9cd --- /dev/null +++ b/tests/ui/codegen/vfe-drop-meta-sized.rs @@ -0,0 +1,58 @@ +//@ run-pass +//@ compile-flags: -Zvirtual-function-elimination -Clto=fat -Copt-level=3 -Zverify-llvm-ir +//@ no-prefer-dynamic +//@ ignore-backends: gcc +#![feature(sized_hierarchy)] +#![no_main] + +// Check that the Drop implementation actually runs. +// +// Before, vtables were not annotated with `!type` annotations for supertraits. Hence, when only +// methods from the supertrait were used, VFE considered the vtable unused and hence the drop +// implementation would be seen as dead and removed. +// +// Checking this on MetaSized is interesting because it is a supertrait to most dyn-compatible +// traits, and it has no methods itself so its vtable is always a prefix of the subtrait's vtable. +// +// This test uses #[no_main] because the standard library is compiled without +// -Zvirtual-function-elimination and therefore virtual calls can miscompile. +// +// Usually this shows up first in `lang_start`, because in `lang_start_internal` we coerce main +// to a `&dyn Fn() -> i32`. But the slot for the function in the vtable is considered dead +// and nulled. The call to the NULL function pointer traps. + +use std::hint::black_box; +use std::marker::MetaSized; +use std::sync::atomic::{AtomicBool, Ordering}; + +static DROPPED: AtomicBool = AtomicBool::new(false); + +#[allow(dead_code)] +trait T { + fn method(&self) -> i32; +} + +struct S; + +impl T for S { + fn method(&self) -> i32 { + 4 + } +} + +impl Drop for S { + fn drop(&mut self) { + DROPPED.store(true, Ordering::Relaxed); + } +} + +#[unsafe(no_mangle)] +extern "C" fn main() -> i32 { + let t: Box = Box::new(S); + let m: Box = black_box(t); + drop(m); + + assert!(DROPPED.load(Ordering::Relaxed)); + + 0 +}