Rollup merge of #138599 - adwinwhite:recursive-overflow, r=wesleywiser

avoid overflow when generating debuginfo for expanding recursive types

Fixes #135093
Fixes #121538
Fixes #107362
Fixes #100618
Fixes #115994

The overflow happens because expanding recursive types keep creating new nested types when recurring into sub fields.
I fixed that by returning an empty stub node when expanding recursion is detected.

Author: matthiaskrgr

compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs

@@ -247,6 +247,16 @@ pub(super) fn stub<'ll, 'tcx>(
     StubInfo { metadata, unique_type_id }
 }
 
+struct AdtStackPopGuard<'ll, 'tcx, 'a> {
+    cx: &'a CodegenCx<'ll, 'tcx>,
+}
+
+impl<'ll, 'tcx, 'a> Drop for AdtStackPopGuard<'ll, 'tcx, 'a> {
+    fn drop(&mut self) {
+        debug_context(self.cx).adt_stack.borrow_mut().pop();
+    }
+}
+
 /// This function enables creating debuginfo nodes that can recursively refer to themselves.
 /// It will first insert the given stub into the type map and only then execute the `members`
 /// and `generics` closures passed in. These closures have access to the stub so they can
@@ -261,6 +271,44 @@ pub(super) fn build_type_with_children<'ll, 'tcx>(
 ) -> DINodeCreationResult<'ll> {
     assert_eq!(debug_context(cx).type_map.di_node_for_unique_id(stub_info.unique_type_id), None);
 
+    let mut _adt_stack_pop_guard = None;
+    if let UniqueTypeId::Ty(ty, ..) = stub_info.unique_type_id
+        && let ty::Adt(adt_def, args) = ty.kind()
+    {
+        let def_id = adt_def.did();
+        // If any sub type reference the original type definition and the sub type has a type
+        // parameter that strictly contains the original parameter, the original type is a recursive
+        // type that can expanding indefinitely. Example,
+        // ```
+        // enum Recursive<T> {
+        //     Recurse(*const Recursive<Wrap<T>>),
+        //     Item(T),
+        // }
+        // ```
+        let is_expanding_recursive =
+            debug_context(cx).adt_stack.borrow().iter().any(|(parent_def_id, parent_args)| {
+                if def_id == *parent_def_id {
+                    args.iter().zip(parent_args.iter()).any(|(arg, parent_arg)| {
+                        if let (Some(arg), Some(parent_arg)) = (arg.as_type(), parent_arg.as_type())
+                        {
+                            arg != parent_arg && arg.contains(parent_arg)
+                        } else {
+                            false
+                        }
+                    })
+                } else {
+                    false
+                }
+            });
+        if is_expanding_recursive {
+            // FIXME: indicate that this is an expanding recursive type in stub metadata?
+            return DINodeCreationResult::new(stub_info.metadata, false);
+        } else {
+            debug_context(cx).adt_stack.borrow_mut().push((def_id, args));
+            _adt_stack_pop_guard = Some(AdtStackPopGuard { cx });
+        }
+    }
+
     debug_context(cx).type_map.insert(stub_info.unique_type_id, stub_info.metadata);
 
     let members: SmallVec<_> =

compiler/rustc_codegen_llvm/src/debuginfo/mod.rs

@@ -66,6 +66,7 @@ pub(crate) struct CodegenUnitDebugContext<'ll, 'tcx> {
     created_files: RefCell<UnordMap<Option<(StableSourceFileId, SourceFileHash)>, &'ll DIFile>>,
 
     type_map: metadata::TypeMap<'ll, 'tcx>,
+    adt_stack: RefCell<Vec<(DefId, GenericArgsRef<'tcx>)>>,
     namespace_map: RefCell<DefIdMap<&'ll DIScope>>,
     recursion_marker_type: OnceCell<&'ll DIType>,
 }
@@ -80,6 +81,7 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> {
             builder,
             created_files: Default::default(),
             type_map: Default::default(),
+            adt_stack: Default::default(),
             namespace_map: RefCell::new(Default::default()),
             recursion_marker_type: OnceCell::new(),
         }

tests/crashes/100618.rs

@@ -4,7 +4,7 @@
 // gdb-command:run
 
 // Test whether compiling a recursive enum definition crashes debug info generation. The test case
-// is taken from issue #11083.
+// is taken from issue #11083 and #135093.
 
 #![allow(unused_variables)]
 #![feature(omit_gdb_pretty_printer_section)]
@@ -18,6 +18,21 @@ struct WindowCallbacks<'a> {
     pos_callback: Option<Box<FnMut(&Window, i32, i32) + 'a>>,
 }
 
+enum ExpandingRecursive<T> {
+    Recurse(Indirect<T>),
+    Item(T),
+}
+
+struct Indirect<U> {
+    rec: *const ExpandingRecursive<Option<U>>,
+}
+
+
 fn main() {
     let x = WindowCallbacks { pos_callback: None };
+
+    // EXPANDING RECURSIVE
+    let expanding_recursive: ExpandingRecursive<u64> = ExpandingRecursive::Recurse(Indirect {
+        rec: &ExpandingRecursive::Item(Option::Some(42)),
+    });
 }

tests/crashes/115994.rs

@@ -1,4 +1,3 @@
-//@ known-bug: #107362
 //@ compile-flags: -Cdebuginfo=2
 
 pub trait Functor