disallow circular struct references (#8851)

* detect and fail for circular struct dependencies

* pr comments

* pr comment

include/flatbuffers/idl.h

diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h
index 06f93f3..9e58d4b 100644
--- a/include/flatbuffers/idl.h
+++ b/include/flatbuffers/idl.h
@@ -395,13 +395,20 @@
 };
 
 struct StructDef : public Definition {
+  enum class CycleStatus {
+    NotChecked,
+    InProgress,
+    Checked,
+  };
+
   StructDef()
       : fixed(false),
         predecl(true),
         sortbysize(true),
         has_key(false),
         minalign(1),
-        bytesize(0) {}
+        bytesize(0),
+        cycle_status{CycleStatus::NotChecked} {}
 
   void PadLastField(size_t min_align) {
     auto padding = PaddingBytes(bytesize, min_align);
@@ -423,6 +430,8 @@
   size_t minalign;  // What the whole object needs to be aligned to.
   size_t bytesize;  // Size if fixed.
 
+  CycleStatus cycle_status;  // used for determining if we have circular references
+
   flatbuffers::unique_ptr<std::string> original_location;
   std::vector<voffset_t> reserved_ids;
 };
@@ -1101,6 +1110,8 @@
   // others includes.
   std::vector<IncludedFile> GetIncludedFiles() const;
 
+  bool HasCircularStructDependency();
+
  private:
   class ParseDepthGuard;
 

src/flatc.cpp

diff --git a/src/flatc.cpp b/src/flatc.cpp
index 6a57085..ee34044 100644
--- a/src/flatc.cpp
+++ b/src/flatc.cpp
@@ -927,6 +927,9 @@
         auto err = parser->ConformTo(conform_parser);
         if (!err.empty()) Error("schemas don\'t conform: " + err, false);
       }
+      if (parser->HasCircularStructDependency()) {
+        Error("schema has circular struct dependencies: " + filename, false);
+      }
       if (options.schema_binary || opts.binary_schema_gen_embed) {
         parser->Serialize();
       }

src/idl_parser.cpp

diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp
index 0aaccbb..569c767 100644
--- a/src/idl_parser.cpp
+++ b/src/idl_parser.cpp
@@ -2755,6 +2755,42 @@
   return {it->second.cbegin(), it->second.cend()};
 }
 
+bool Parser::HasCircularStructDependency() {
+  std::function<bool(StructDef*)> visit =
+      [&](StructDef* struct_def) {
+        // Only consider fixed structs and structs we have yet to check
+        if (!struct_def->fixed || struct_def->cycle_status == StructDef::CycleStatus::Checked) {
+          return false;
+        }
+
+        if (struct_def->cycle_status == StructDef::CycleStatus::InProgress) {
+          // cycle found
+          return true;
+        }
+
+        struct_def->cycle_status = StructDef::CycleStatus::InProgress;
+
+        for (const auto& field : struct_def->fields.vec) {
+          if (field->value.type.base_type == BASE_TYPE_STRUCT) {
+            if (visit(field->value.type.struct_def)) {
+              return true;  // Cycle detected in recursion.
+            }
+          }
+        }
+
+        struct_def->cycle_status = StructDef::CycleStatus::Checked;
+        return false;  // No cycle detected.
+      };
+
+  for (const auto& struct_def : structs_.vec) {
+    if (visit(struct_def)) {
+      return true;  // Cycle detected.
+    }
+  }
+
+  return false;  // No cycle detected.
+}
+
 bool Parser::SupportsOptionalScalars(const flatbuffers::IDLOptions& opts) {
   static FLATBUFFERS_CONSTEXPR unsigned long supported_langs =
       IDLOptions::kRust | IDLOptions::kSwift | IDLOptions::kLobster |

tests/flatc/flatc_schema_tests.py

diff --git a/tests/flatc/flatc_schema_tests.py b/tests/flatc/flatc_schema_tests.py
index ebd5a79..f915208 100644
--- a/tests/flatc/flatc_schema_tests.py
+++ b/tests/flatc/flatc_schema_tests.py
@@ -71,3 +71,12 @@
         schema_json["enums"][0]["values"][2]["attributes"][1]["value"]
         == "Value 3 (deprecated)"
     )
+
+  def CircularStructDependency(self):
+    try:
+      flatc(["-c", "circular_struct_dependency.fbs"])
+      assert False, "Expected flatc to fail on circular struct dependency"
+    except subprocess.CalledProcessError:
+      pass
+    
+    flatc(["-c", "circular_table.fbs"])
\ No newline at end of file