fix a bug in circular type detection (#35275)

Author: JeffBezanson

src/datatype.c

@@ -279,25 +279,25 @@ int jl_pointer_egal(jl_value_t *t)
     return 0;
 }
 
-static int references_name(jl_value_t *p, jl_typename_t *name) JL_NOTSAFEPOINT
+static int references_name(jl_value_t *p, jl_typename_t *name, int affects_layout) JL_NOTSAFEPOINT
 {
     if (jl_is_uniontype(p))
-        return references_name(((jl_uniontype_t*)p)->a, name) ||
-               references_name(((jl_uniontype_t*)p)->b, name);
+        return references_name(((jl_uniontype_t*)p)->a, name, affects_layout) ||
+               references_name(((jl_uniontype_t*)p)->b, name, affects_layout);
     if (jl_is_unionall(p))
-        return references_name((jl_value_t*)((jl_unionall_t*)p)->var, name) ||
-               references_name(((jl_unionall_t*)p)->body, name);
+        return references_name((jl_value_t*)((jl_unionall_t*)p)->var, name, 0) ||
+               references_name(((jl_unionall_t*)p)->body, name, affects_layout);
     if (jl_is_typevar(p))
-        return references_name(((jl_tvar_t*)p)->ub, name) ||
-               references_name(((jl_tvar_t*)p)->lb, name);
+        return references_name(((jl_tvar_t*)p)->ub, name, 0) ||
+               references_name(((jl_tvar_t*)p)->lb, name, 0);
     if (jl_is_datatype(p)) {
-        if (((jl_datatype_t*)p)->name == name)
+        jl_datatype_t *dp = (jl_datatype_t*)p;
+        if (affects_layout && dp->name == name)
             return 1;
-        if (((jl_datatype_t*)p)->layout && jl_datatype_nfields(p) == 0)
-            return 0;
+        affects_layout = dp->types == NULL || jl_svec_len(dp->types) != 0;
         size_t i, l = jl_nparams(p);
         for (i = 0; i < l; i++) {
-            if (references_name(jl_tparam(p, i), name))
+            if (references_name(jl_tparam(p, i), name, affects_layout))
                 return 1;
         }
     }
@@ -390,7 +390,7 @@ void jl_compute_field_offsets(jl_datatype_t *st)
         size_t i, nf = jl_svec_len(w->types);
         for (i = 0; i < nf; i++) {
             jl_value_t *fld = jl_svecref(w->types, i);
-            if (references_name(fld, w->name)) {
+            if (references_name(fld, w->name, 1)) {
                 isinlinealloc = 0;
                 isbitstype = 0;
                 break;

test/core.jl

@@ -7191,3 +7191,10 @@ end
 @test_throws ErrorException f34482()
 @test_throws TypeError g34482()
 @test_throws TypeError h34482()
+
+struct NFANode34126
+    edges::Vector{Tuple{Nothing,NFANode34126}}
+    NFANode34126() = new(Tuple{Nothing,NFANode34126}[])
+end
+
+@test repr(NFANode34126()) == "$NFANode34126(Tuple{Nothing,$NFANode34126}[])"