fix(serialization): references inlining

src/Microsoft.OpenApi/Extensions/OpenAPIWriterExtensions.cs

@@ -1,6 +1,6 @@
 namespace Microsoft.OpenApi
 {
     internal static class OpenAPIWriterExtensions
     {
         /// <summary>
         /// Temporary extension method until we add Settings property to IOpenApiWriter in next major version
@@ -15,5 +15,13 @@
             }
             return new();
         }
+
+        internal static void CountInlineReference(this IOpenApiWriter openApiWriter)
+        {
+            if (openApiWriter is OpenApiWriterBase @base)
+            {
+                @base.CountInlineReference();
+            }
+        }
     }
 }

src/Microsoft.OpenApi/Models/OpenApiHeader.cs

@@ -98,7 +98,7 @@ public virtual void SerializeAsV3(IOpenApiWriter writer)
             SerializeInternal(writer, OpenApiSpecVersion.OpenApi3_0, (writer, element) => element.SerializeAsV3(writer));
         }
 
-        internal void SerializeInternal(IOpenApiWriter writer, OpenApiSpecVersion version, 
+        internal void SerializeInternal(IOpenApiWriter writer, OpenApiSpecVersion version,
             Action<IOpenApiWriter, IOpenApiSerializable> callback)
         {
             Utils.CheckArgumentNull(writer);
@@ -175,7 +175,12 @@ public virtual void SerializeAsV2(IOpenApiWriter writer)
             writer.WriteProperty(OpenApiConstants.AllowReserved, AllowReserved, false);
 
             // schema
-            var targetSchema = Schema switch {
+            if (Schema is OpenApiSchemaReference)
+            {
+                writer.CountInlineReference();
+            }
+            var targetSchema = Schema switch
+            {
                 OpenApiSchemaReference schemaReference => schemaReference.RecursiveTarget,
                 OpenApiSchema schema => schema,
                 _ => null,

src/Microsoft.OpenApi/Models/OpenApiParameter.cs

@@ -33,7 +33,7 @@
         public bool Deprecated { get; set; }
 
         /// <inheritdoc/>
         [Obsolete("Use of AllowEmptyValue is not recommended and it is likely to be removed in a later revision.")]
         public bool AllowEmptyValue { get; set; }
 
         /// <inheritdoc/>
@@ -113,11 +113,11 @@
             SerializeInternal(writer, OpenApiSpecVersion.OpenApi3_0, (writer, element) => element.SerializeAsV3(writer));
         }
 
-        internal void SerializeInternal(IOpenApiWriter writer, OpenApiSpecVersion version, 
+        internal void SerializeInternal(IOpenApiWriter writer, OpenApiSpecVersion version,
             Action<IOpenApiWriter, IOpenApiSerializable> callback)
         {
             Utils.CheckArgumentNull(writer);
-            
+
             // Validate that Cookie style is only used in OpenAPI 3.2 and later
             if (Style == ParameterStyle.Cookie && version < OpenApiSpecVersion.OpenApi3_2)
             {
@@ -201,7 +201,7 @@
         /// </summary>
         /// <param name="writer">Writer to use for the serialization</param>
         /// <param name="extensionsClone">Extensions clone</param>
         internal virtual void WriteRequestBodySchemaForV2(IOpenApiWriter writer, Dictionary<string, IOpenApiExtension>? extensionsClone)
         {
             // In V2 parameter's type can't be a reference to a custom object schema or can't be of type object
             // So in that case map the type as string.
@@ -228,7 +228,12 @@
                 // uniqueItems
                 // enum
                 // multipleOf
-                var targetSchema = Schema switch {
+                if (Schema is OpenApiSchemaReference)
+                {
+                    writer.CountInlineReference();
+                }
+                var targetSchema = Schema switch
+                {
                     OpenApiSchemaReference schemaReference => schemaReference.RecursiveTarget,
                     OpenApiSchema schema => schema,
                     _ => null,
@@ -253,7 +258,7 @@
 
                 if (this.In == ParameterLocation.Query && Schema?.Type == JsonSchemaType.Array)
                 {
                     if (this.Style == ParameterStyle.Form && this.Explode == true)
                     {
                         writer.WriteProperty("collectionFormat", "multi");
                     }
@@ -273,7 +278,7 @@
         public virtual void SerializeAsV2(IOpenApiWriter writer)
         {
             Utils.CheckArgumentNull(writer);
-            
+
             // Validate that Cookie style is only used in OpenAPI 3.2 and later
             if (Style == ParameterStyle.Cookie)
             {

src/Microsoft.OpenApi/Models/References/BaseOpenApiReferenceHolder.cs

@@ -9,7 +9,7 @@
 /// <typeparam name="T">The concrete class implementation type for the model.</typeparam>
 /// <typeparam name="U">The interface type for the model.</typeparam>
 /// <typeparam name="V">The type for the reference holding the additional fields and annotations</typeparam>
 public abstract class BaseOpenApiReferenceHolder<T, U, V> : IOpenApiReferenceHolder<T, U, V> where T : class, IOpenApiReferenceable, U where U : IOpenApiReferenceable, IOpenApiSerializable where V : BaseOpenApiReference, new()
 {
     [ThreadStatic]
     private static HashSet<BaseOpenApiReferenceHolder<T, U, V>>? t_activeReferenceAccesses;
@@ -70,7 +70,7 @@
         return ExecuteWithReferenceGuard(ref t_activeReferenceAccesses, holder, action);
     }
 
     private static TResult ExecuteWithTargetActionGuard<TResult>(
         BaseOpenApiReferenceHolder<T, U, V> holder,
         Func<TResult> action)
     {
@@ -137,7 +137,7 @@
 
     private sealed class ReferenceHolderComparer : IEqualityComparer<BaseOpenApiReferenceHolder<T, U, V>>
     {
         internal static ReferenceHolderComparer Instance { get; } = new();
 
         public bool Equals(BaseOpenApiReferenceHolder<T, U, V>? x, BaseOpenApiReferenceHolder<T, U, V>? y)
         {
@@ -163,7 +163,7 @@
     protected BaseOpenApiReferenceHolder(BaseOpenApiReferenceHolder<T, U, V> source)
     {
         Utils.CheckArgumentNull(source);
         Reference = CopyReference(source.Reference);
         //no need to copy summary and description as if they are not overridden, they will be fetched from the target
         //if they are, the reference copy will handle it
     }
@@ -207,7 +207,7 @@
     /// <inheritdoc/>
     public virtual void SerializeAsV3(IOpenApiWriter writer)
     {
-        if (!writer.GetSettings().ShouldInlineReference(Reference) 
+        if (!writer.GetSettings().ShouldInlineReference(Reference)
             || Reference.Type == ReferenceType.Tag) // tags are held as references need to drop in.
         {
             Reference.SerializeAsV3(writer);
@@ -267,6 +267,7 @@
         Action<IOpenApiWriter, U> action)
     {
         Utils.CheckArgumentNull(writer);
+        writer.CountInlineReference();
         ApplyToTarget(element => action(writer, element));
     }
 }

src/Microsoft.OpenApi/Models/References/OpenApiMediaTypeReference.cs

@@ -77,11 +77,13 @@ protected override BaseOpenApiReference CopyReference(BaseOpenApiReference sourc
         /// <inheritdoc/>
         public override void SerializeAsV31(IOpenApiWriter writer)
         {
+            writer.CountInlineReference();
             CopyReferenceAsTargetElementWithOverrides(new OpenApiMediaType()).SerializeAsV31(writer);
         }
         /// <inheritdoc/>
         public override void SerializeAsV3(IOpenApiWriter writer)
         {
+            writer.CountInlineReference();
             CopyReferenceAsTargetElementWithOverrides(new OpenApiMediaType()).SerializeAsV3(writer);
         }
     }

src/Microsoft.OpenApi/Models/References/OpenApiRequestBodyReference.cs

@@ -64,6 +64,7 @@ public override void SerializeAsV2(IOpenApiWriter writer)
         {
             if (writer.GetSettings().ShouldInlineReference(Reference))
             {
+                writer.CountInlineReference();
                 return GetFromTarget(target => target.ConvertToBodyParameter(writer));
             }
 
@@ -74,6 +75,7 @@ public override void SerializeAsV2(IOpenApiWriter writer)
         {
             if (writer.GetSettings().ShouldInlineReference(Reference))
             {
+                writer.CountInlineReference();
                 return GetFromTarget(target => target.ConvertToFormDataParameters(writer));
             }
 

src/Microsoft.OpenApi/Models/References/OpenApiSchemaReference.cs

@@ -167,7 +167,7 @@
         /// <inheritdoc/>
         public OpenApiDiscriminator? Discriminator { get => Reference.Discriminator ?? GetFromTarget(static target => target.Discriminator); set => Reference.Discriminator = value; }
         /// <inheritdoc/>
         [Obsolete("Use Examples instead.")]
         public JsonNode? Example { get => Reference.Example ?? GetFromTarget(static target => target.Example); set => Reference.Example = value; }
         /// <inheritdoc/>
         public IList<JsonNode>? Examples
@@ -256,8 +256,14 @@
             }
             else
             {
-                SerializeInternal(writer, (w, element) => action(w, element));
-                writer.GetSettings().LoopDetector.PopLoop<IOpenApiSchema>();
+                try
+                {
+                    SerializeInternal(writer, (w, element) => action(w, element));
+                }
+                finally
+                {
+                    writer.GetSettings().LoopDetector.PopLoop<IOpenApiSchema>();
+                }
             }
 
         }

src/Microsoft.OpenApi/Properties/SRResource.resx

@@ -132,6 +132,9 @@
   <data name="IndentationLevelInvalid" xml:space="preserve">
     <value>Indentation level cannot be lower than 0.</value>
   </data>
+  <data name="MaximumInlineReferenceCountExceeded" xml:space="preserve">
+    <value>The maximum number of inline references ({0}) has been exceeded.</value>
+  </data>
   <data name="InputItemShouldBeType" xml:space="preserve">
     <value>The input item should be in type of '{0}'.</value>
   </data>

src/Microsoft.OpenApi/PublicAPI.Unshipped.txt

@@ -1,2 +1,4 @@
 #nullable enable
 Microsoft.OpenApi.OpenApiDocument.GetOperationById(string! operationId) -> Microsoft.OpenApi.OpenApiOperation?
+Microsoft.OpenApi.OpenApiWriterSettings.MaxInlineReferenceCount.get -> int
+Microsoft.OpenApi.OpenApiWriterSettings.MaxInlineReferenceCount.set -> void

src/Microsoft.OpenApi/Writers/OpenApiWriterBase.cs

@@ -35,6 +35,7 @@
         /// Number which specifies the level of indentation.
         /// </summary>
         private int _indentLevel;
+        private int _inlineReferenceCount;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="OpenApiWriterBase"/> class.
@@ -70,6 +71,20 @@
         /// </summary>
         protected TextWriter Writer { get; }
 
+        internal void CountInlineReference()
+        {
+            if (_inlineReferenceCount >= Settings.MaxInlineReferenceCount)
+            {
+                throw new OpenApiWriterException(
+                    string.Format(
+                        CultureInfo.InvariantCulture,
+                        SRResource.MaximumInlineReferenceCountExceeded,
+                        Settings.MaxInlineReferenceCount));
+            }
+
+            _inlineReferenceCount++;
+        }
+
         /// <summary>
         /// Write start object.
         /// </summary>
@@ -130,7 +145,7 @@
         /// Write float value.
         /// </summary>
         /// <param name="value">The float value.</param>
         public virtual void WriteValue(float value)
         {
             WriteValueSeparator();
             Writer.Write(value);
@@ -319,7 +334,7 @@
                 Writer.Write(IndentationString);
             }
         }
-        
+
         /// <summary>
         /// Get current scope.
         /// </summary>

src/Microsoft.OpenApi/Writers/OpenApiWriterSettings.cs

@@ -5,7 +5,10 @@
     /// </summary>
     public class OpenApiWriterSettings
     {
+        private const int DefaultMaxInlineReferenceCount = 10_000;
+
         internal LoopDetector LoopDetector { get; } = new();
+
         /// <summary>
         /// Indicates if local references should be rendered as an inline object
         /// </summary>
@@ -16,6 +19,12 @@ public class OpenApiWriterSettings
         /// </summary>
         public bool InlineExternalReferences { get; set; }
 
+        /// <summary>
+        /// Gets or sets the maximum number of references that can be expanded inline by a writer.
+        /// </summary>
+        /// <remarks>The default is 10,000.</remarks>
+        public int MaxInlineReferenceCount { get; set; } = DefaultMaxInlineReferenceCount;
+
         internal bool ShouldInlineReference(BaseOpenApiReference reference)
         {
             return (reference.IsLocal && InlineLocalReferences)