[cxx-interop] Fix crashing on recursive enums

Introduce a cache that helps cutting the recursion when we process a
type that we already visited before but did not finish processing yet.

Fixes #85361

rdar://164153038

Author: Gabor Horvath

lib/PrintAsClang/DeclAndTypePrinter.h

@@ -15,10 +15,12 @@
 
 #include "OutputLanguageMode.h"
 
+#include "PrintClangFunction.h"
 #include "swift/AST/Decl.h"
 #include "swift/AST/Module.h"
 #include "swift/AST/Type.h"
 // for OptionalTypeKind
+#include "swift/AST/TypeRepr.h"
 #include "swift/ClangImporter/ClangImporter.h"
 #include "llvm/ADT/StringSet.h"
 
@@ -56,6 +58,7 @@ class DeclAndTypePrinter {
 private:
   class Implementation;
   friend class Implementation;
+  friend class DeclAndTypeClangFunctionPrinter;
 
   ModuleDecl &M;
   raw_ostream &os;
@@ -69,6 +72,7 @@ class DeclAndTypePrinter {
   bool requiresExposedAttribute;
   llvm::StringSet<> &exposedModules;
   OutputLanguageMode outputLang;
+  llvm::DenseMap<Type, std::optional<ClangRepresentation>> typeRepresentations;
 
   /// The name 'CFTypeRef'.
   ///

lib/PrintAsClang/PrintClangFunction.cpp

@@ -1895,13 +1895,27 @@ ClangRepresentation DeclAndTypeClangFunctionPrinter::getTypeRepresentation(
     PrimitiveTypeMapping &typeMapping,
     SwiftToClangInteropContext &interopContext, DeclAndTypePrinter &declPrinter,
     const ModuleDecl *emittedModule, Type ty) {
+  auto [it, inserted] = declPrinter.typeRepresentations.try_emplace(ty);
+  // If we already seen the type but do not have a representation yet assume
+  // representable for now. This can happen for recursive types like:
+  //
+  //    public enum E {
+  //      case foo([E?])
+  //    }
+  //
+  // We make a decision about these types when the function that first
+  // encountered them returns.
+  if (!inserted)
+    return it->second ? (*it->second) : ClangRepresentation::representable;
   CFunctionSignatureTypePrinterModifierDelegate delegate;
   CFunctionSignatureTypePrinter typePrinter(
       llvm::nulls(), llvm::nulls(), typeMapping, OutputLanguageMode::Cxx,
       interopContext, delegate, emittedModule, declPrinter,
       FunctionSignatureTypeUse::TypeReference);
-  return typePrinter.visit(ty, OptionalTypeKind::OTK_None,
-                           /*isInOutParam=*/false);
+  auto result = typePrinter.visit(ty, OptionalTypeKind::OTK_None,
+                                  /*isInOutParam=*/false);
+  declPrinter.typeRepresentations[ty] = result;
+  return result;
 }
 
 void DeclAndTypeClangFunctionPrinter::printTypeName(

test/Interop/SwiftToCxx/enums/recursive-enum.swift

@@ -0,0 +1,8 @@
+// RUN: %empty-directory(%t)
+// RUN: %target-swift-frontend %s -module-name Enums -clang-header-expose-decls=all-public -typecheck -verify -emit-clang-header-path %t/enums.h
+
+// RUN: %check-interop-cxx-header-in-clang(%t/enums.h -Wno-unused-private-field -Wno-unused-function -DSWIFT_CXX_INTEROP_HIDE_STL_OVERLAY)
+
+public enum E {
+  case foo([E?])
+}