Detect cyclic subtypes during generalization in the new solver

compiler/rustc_infer/src/infer/relate/generalize.rs

@@ -557,8 +557,22 @@ impl<'tcx> TypeRelation<TyCtxt<'tcx>> for Generalizer<'_, 'tcx> {
                 let mut inner = self.infcx.inner.borrow_mut();
                 let vid = inner.type_variables().root_var(vid);
                 if TermVid::Ty(vid) == self.root_vid {
-                    // If sub-roots are equal, then `root_vid` and
-                    // `vid` are related via subtyping.
+                    // `root_vid` occurs in the term we're generalizing, so
+                    // instantiating it would result in a cyclic type.
+                    Err(self.cyclic_term_error())
+                } else if self.infcx.next_trait_solver()
+                    && let TermVid::Ty(root_vid) = self.root_vid
+                    && inner.type_variables().sub_unification_table_root_var(vid)
+                        == inner.type_variables().sub_unification_table_root_var(root_vid)
+                {
+                    // If the sub-unification roots of `root_vid` and `vid` are equal, then
+                    // the two variables are related via subtyping. As subtyping never
+                    // changes the structure of a type, instantiating `root_vid` with a
+                    // type containing `vid` would also result in a cyclic type.
+                    //
+                    // We only check this with the new trait solver. Doing so with the old
+                    // solver is unsound as its evaluation cache does not consider the
+                    // `sub_unification_table`.
                     Err(self.cyclic_term_error())
                 } else {
                     let probe = inner.type_variables().probe(vid);

tests/ui/inference/cyclic-subtype-extend-157619.current.stderr

@@ -0,0 +1,9 @@
+error[E0275]: overflow evaluating whether `&_` is well-formed
+  --> $DIR/cyclic-subtype-extend-157619.rs:14:23
+   |
+LL |     let mut samples = Vec::new();
+   |                       ^^^
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0275`.

tests/ui/inference/cyclic-subtype-extend-157619.next.stderr

@@ -0,0 +1,17 @@
+error[E0277]: the trait bound `Vec<(&_,)>: Extend<(_,)>` is not satisfied
+  --> $DIR/cyclic-subtype-extend-157619.rs:18:13
+   |
+LL |     samples.extend(packet_buf.iter().map(|&x| (x,)));
+   |             ^^^^^^ the trait `Extend<(_,)>` is not implemented for `Vec<(&_,)>`
+   |
+help: `Vec<T, A>` implements trait `Extend<A>`
+  --> $SRC_DIR/alloc/src/vec/mod.rs:LL:COL
+   |
+   = note: `Extend<T>`
+  ::: $SRC_DIR/alloc/src/vec/mod.rs:LL:COL
+   |
+   = note: `Extend<&T>`
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0277`.

tests/ui/inference/cyclic-subtype-extend-157619.rs

@@ -0,0 +1,20 @@
+// Regression test for #157619. A `Subtype` obligation between two inference
+// variables where one is then instantiated with a type containing the other
+// used to make the inferred types grow without bound, resulting in an
+// overflow error whose span and message are unrelated to the actual cause.
+//
+// With `-Znext-solver` we now use the sub-unification table to eagerly
+// detect the cyclic type when generalizing.
+
+//@ revisions: current next
+//@ ignore-compare-mode-next-solver (explicit revisions)
+//@[next] compile-flags: -Znext-solver
+
+fn main() {
+    let mut samples = Vec::new();
+    //[current]~^ ERROR overflow evaluating whether `&_` is well-formed
+    let packet_buf = Vec::new();
+    samples.extend(packet_buf.iter().map(|x| (x,)));
+    samples.extend(packet_buf.iter().map(|&x| (x,)));
+    //[next]~^ ERROR the trait bound `Vec<(&_,)>: Extend<(_,)>` is not satisfied
+}

tests/ui/inference/cyclic-subtype-push-128397.current.stderr

@@ -0,0 +1,9 @@
+error[E0275]: overflow evaluating whether `&_` is well-formed
+  --> $DIR/cyclic-subtype-push-128397.rs:16:13
+   |
+LL |     let x = v.last().unwrap();
+   |             ^^^^^^^^
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0275`.

tests/ui/inference/cyclic-subtype-push-128397.next.stderr

@@ -0,0 +1,14 @@
+error[E0308]: mismatched types
+  --> $DIR/cyclic-subtype-push-128397.rs:18:12
+   |
+LL |     v.push(x);
+   |       ---- ^ recursive type with infinite-size name
+   |       |
+   |       arguments to this method are incorrect
+   |
+note: method defined here
+  --> $SRC_DIR/alloc/src/vec/mod.rs:LL:COL
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0308`.

tests/ui/inference/cyclic-subtype-push-128397.rs

@@ -0,0 +1,20 @@
+// Regression test for #128397. Pushing a borrow of a collection's element
+// back into the collection requires the element type to contain its own
+// reference. This used to make the inferred type grow without bound,
+// resulting in an overflow error whose span and message are unrelated to
+// the actual cause.
+//
+// With `-Znext-solver` we now use the sub-unification table to eagerly
+// detect the cyclic type when generalizing.
+
+//@ revisions: current next
+//@ ignore-compare-mode-next-solver (explicit revisions)
+//@[next] compile-flags: -Znext-solver
+
+fn main() {
+    let mut v = vec![];
+    let x = v.last().unwrap();
+    //[current]~^ ERROR overflow evaluating whether `&_` is well-formed
+    v.push(x);
+    //[next]~^ ERROR mismatched types
+}