Replace JSON.NET with System.Text.Json across the codebase

Remove all Newtonsoft.Json serialization from the application layer, replacing
it with System.Text.Json (STJ). NEST still brings in Newtonsoft transitively,
but all application-level serialization now uses STJ exclusively.

Key changes:
- Add ElasticSystemTextJsonSerializer as custom IElasticsearchSerializer for NEST
- Add EmptyCollectionModifier to omit empty collections during serialization
- Add ObjectToInferredTypesConverter to handle JObject/JToken from NEST reads
- Add JsonNodeExtensions as STJ equivalents of JObject helpers for event upgraders
- Add IJsonOnDeserialized to Event model to merge [JsonExtensionData] into Data dict
- Add [JsonPropertyName] attributes to V1 webhook models for PascalCase compat
- Migrate all event upgraders from JObject to JsonObject (System.Text.Json.Nodes)
- Migrate all plugins from ISerializer/JsonSerializerOptions DI injection
- Use case-insensitive deserialization for DataDictionary.GetValue<T>() from JsonElement
- Use semantic comparison (JsonNode.DeepEquals) in tests for fixture validation
- Remove DataObjectConverter, ElasticJsonNetSerializer, and related Newtonsoft classes
- Remove Foundatio.JsonNet, NEST.JsonNetSerializer, FluentRest.NewtonsoftJson packages

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: niemyjski

AGENTS.md

@@ -82,3 +82,26 @@ pr-reviewer → security pre-screen (before build!) → dependency audit
 - Never commit secrets — use environment variables
 - NuGet feeds are in `NuGet.Config` — don't add sources
 - Prefer additive documentation updates — don't replace strategic docs wholesale, extend them
+
+## Serialization Architecture
+
+The project uses **System.Text.Json (STJ)** exclusively. NEST still brings in Newtonsoft.Json transitively, but all application-level serialization uses STJ:
+
+| Component      | Serializer                        | Notes                                       |
+| -------------- | --------------------------------- | -------------------------------------------- |
+| Elasticsearch  | `ElasticSystemTextJsonSerializer` | Custom `IElasticsearchSerializer` using STJ  |
+| Event Upgrader | `System.Text.Json.Nodes`          | JsonObject/JsonArray for mutable DOM         |
+| Data Storage   | `SystemTextJsonSerializer`        | Via Foundatio's STJ support                  |
+| API            | STJ (built-in)                    | ASP.NET Core default with custom options     |
+
+**Key files:**
+
+- `ElasticSystemTextJsonSerializer.cs` - Custom `IElasticsearchSerializer` for NEST
+- `JsonNodeExtensions.cs` - STJ equivalents of JObject helpers
+- `ObjectToInferredTypesConverter.cs` - Handles JObject/JToken from NEST during STJ serialization
+- `V*_EventUpgrade.cs` - Event version upgraders using JsonObject
+
+**Security:**
+
+- Safe JSON encoding used everywhere (escapes `<`, `>`, `&`, `'` for XSS protection)
+- No `UnsafeRelaxedJsonEscaping` in the codebase

src/Exceptionless.Core/Bootstrapper.cs

@@ -24,7 +24,6 @@
 using Exceptionless.Core.Services;
 using Exceptionless.Core.Utility;
 using Exceptionless.Core.Validation;
-using Exceptionless.Serializer;
 using FluentValidation;
 using Foundatio.Caching;
 using Foundatio.Extensions.Hosting.Jobs;
@@ -53,27 +52,7 @@ public class Bootstrapper
 {
     public static void RegisterServices(IServiceCollection services, AppOptions appOptions)
     {
-        // PERF: Work towards getting rid of JSON.NET.
-        Newtonsoft.Json.JsonConvert.DefaultSettings = () => new Newtonsoft.Json.JsonSerializerSettings
-        {
-            DateParseHandling = Newtonsoft.Json.DateParseHandling.DateTimeOffset
-        };
-
-        services.AddSingleton<Newtonsoft.Json.Serialization.IContractResolver>(_ => GetJsonContractResolver());
-        services.AddSingleton<Newtonsoft.Json.JsonSerializerSettings>(s =>
-        {
-            // NOTE: These settings may need to be synced in the Elastic Configuration.
-            var settings = new Newtonsoft.Json.JsonSerializerSettings
-            {
-                MissingMemberHandling = Newtonsoft.Json.MissingMemberHandling.Ignore,
-                DateParseHandling = Newtonsoft.Json.DateParseHandling.DateTimeOffset,
-                ContractResolver = s.GetRequiredService<Newtonsoft.Json.Serialization.IContractResolver>()
-            };
-
-            settings.AddModelConverters(s.GetRequiredService<ILogger<Bootstrapper>>());
-            return settings;
-        });
-
+        // Register System.Text.Json options with Exceptionless defaults (snake_case, null handling)
         services.AddSingleton(_ => new JsonSerializerOptions().ConfigureExceptionlessDefaults());
 
         services.AddSingleton<ISerializer>(s => s.GetRequiredService<ITextSerializer>());
@@ -278,13 +257,6 @@ public static void AddHostedJobs(IServiceCollection services, ILoggerFactory log
         logger.LogWarning("Jobs running in process");
     }
 
-    public static DynamicTypeContractResolver GetJsonContractResolver()
-    {
-        var resolver = new DynamicTypeContractResolver(new LowerCaseUnderscorePropertyNamesContractResolver());
-        resolver.UseDefaultResolverFor(typeof(DataDictionary), typeof(SettingsDictionary), typeof(VersionOnePlugin.VersionOneWebHookStack), typeof(VersionOnePlugin.VersionOneWebHookEvent));
-        return resolver;
-    }
-
     private static IQueue<T> CreateQueue<T>(IServiceProvider container, TimeSpan? workItemTimeout = null) where T : class
     {
         var loggerFactory = container.GetRequiredService<ILoggerFactory>();

src/Exceptionless.Core/Exceptionless.Core.csproj

@@ -24,8 +24,8 @@
     <PackageReference Include="FluentValidation" Version="12.1.1" />
     <PackageReference Include="Foundatio.Extensions.Hosting" Version="13.0.0-beta3.19" />
     <PackageReference Include="Foundatio.JsonNet" Version="13.0.0-beta3.19" />
+    <PackageReference Include="Foundatio.Extensions.Hosting" Version="13.0.0-beta3.5" />
     <PackageReference Include="MiniValidation" Version="0.9.2" />
-    <PackageReference Include="NEST.JsonNetSerializer" Version="7.17.5" />
     <PackageReference Include="Handlebars.Net" Version="2.1.6" />
     <PackageReference Include="McSherry.SemanticVersioning" Version="1.4.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="10.0.5" />

src/Exceptionless.Core/Extensions/DataDictionaryExtensions.cs

@@ -1,35 +1,44 @@
 using System.Text.Json;
 using System.Text.Json.Nodes;
 using Exceptionless.Core.Models;
+using Foundatio.Serializer;
 
 namespace Exceptionless.Core.Extensions;
 
 public static class DataDictionaryExtensions
 {
+    /// <summary>
+    /// Options for deserializing JsonElement values that may use PascalCase or snake_case
+    /// property names. Uses case-insensitive matching without a naming policy so both formats work.
+    /// </summary>
+    private static readonly JsonSerializerOptions CaseInsensitiveOptions = new()
+    {
+        PropertyNameCaseInsensitive = true
+    };
     /// <summary>
     /// Retrieves a typed value from the <see cref="DataDictionary"/>, deserializing if necessary.
     /// </summary>
     /// <typeparam name="T">The target type to deserialize to.</typeparam>
     /// <param name="extendedData">The data dictionary containing the value.</param>
     /// <param name="key">The key of the value to retrieve.</param>
-    /// <param name="options">The JSON serializer options to use for deserialization.</param>
+    /// <param name="serializer">The text serializer to use for deserialization.</param>
     /// <returns>The deserialized value, or <c>default</c> if deserialization fails.</returns>
     /// <exception cref="KeyNotFoundException">Thrown when the key is not found in the dictionary.</exception>
     /// <remarks>
     /// <para>This method handles multiple source formats in priority order:</para>
     /// <list type="number">
     ///   <item><description>Direct type match - returns value directly</description></item>
     ///   <item><description><see cref="JsonDocument"/> - extracts root element and deserializes</description></item>
-    ///   <item><description><see cref="JsonElement"/> - deserializes using provided options</description></item>
-    ///   <item><description><see cref="JsonNode"/> - deserializes using provided options</description></item>
-    ///   <item><description><see cref="Dictionary{TKey,TValue}"/> - re-serializes to JSON then deserializes (for ObjectToInferredTypesConverter output)</description></item>
-    ///   <item><description><see cref="List{T}"/> of objects - re-serializes to JSON then deserializes</description></item>
-    ///   <item><description><see cref="Newtonsoft.Json.Linq.JObject"/> - uses ToObject for Elasticsearch compatibility (data read from Elasticsearch uses JSON.NET)</description></item>
-    ///   <item><description>JSON string - parses and deserializes</description></item>
+    ///   <item><description><see cref="JsonElement"/> - extracts raw JSON and deserializes via ITextSerializer</description></item>
+    ///   <item><description><see cref="JsonNode"/> - extracts JSON string and deserializes via ITextSerializer</description></item>
+    ///   <item><description><see cref="Dictionary{TKey,TValue}"/> - re-serializes to JSON then deserializes via ITextSerializer</description></item>
+    ///   <item><description><see cref="List{T}"/> of objects - re-serializes to JSON then deserializes via ITextSerializer</description></item>
+    ///   <item><description><see cref="Newtonsoft.Json.Linq.JObject"/> - uses ToObject for Elasticsearch compatibility</description></item>
+    ///   <item><description>JSON string - deserializes via ITextSerializer</description></item>
     ///   <item><description>Fallback - attempts type conversion via ToType</description></item>
     /// </list>
     /// </remarks>
-    public static T? GetValue<T>(this DataDictionary extendedData, string key, JsonSerializerOptions options)
+    public static T? GetValue<T>(this DataDictionary extendedData, string key, ITextSerializer serializer)
     {
         if (!extendedData.TryGetValue(key, out object? data))
             throw new KeyNotFoundException($"Key \"{key}\" not found in the dictionary.");
@@ -42,18 +51,46 @@ public static class DataDictionaryExtensions
             data = jsonDocument.RootElement;
 
         // JsonElement (from STJ deserialization when ObjectToInferredTypesConverter wasn't used)
-        if (data is JsonElement jsonElement &&
-            TryDeserialize(jsonElement, options, out T? jsonElementResult))
+        if (data is JsonElement jsonElement)
         {
-            return jsonElementResult;
+            try
+            {
+                // Fast-path for string type
+                if (typeof(T) == typeof(string))
+                {
+                    object? s = jsonElement.ValueKind switch
+                    {
+                        JsonValueKind.String => jsonElement.GetString(),
+                        JsonValueKind.Number => jsonElement.GetRawText(),
+                        JsonValueKind.True => "true",
+                        JsonValueKind.False => "false",
+                        JsonValueKind.Null => null,
+                        _ => jsonElement.GetRawText()
+                    };
+
+                    return (T?)s;
+                }
+
+                // Deserialize directly from JsonElement using case-insensitive matching.
+                // This handles both snake_case (from Elasticsearch) and PascalCase (from
+                // [JsonExtensionData] which preserves original property names).
+                var result = jsonElement.Deserialize<T>(CaseInsensitiveOptions);
+                if (result is not null)
+                    return result;
+            }
+            catch
+            {
+                // Ignored - fall through to next handler
+            }
         }
 
         // JsonNode (JsonObject/JsonArray/JsonValue)
         if (data is JsonNode jsonNode)
         {
             try
             {
-                var result = jsonNode.Deserialize<T>(options);
+                string jsonString = jsonNode.ToJsonString();
+                var result = serializer.Deserialize<T>(jsonString);
                 if (result is not null)
                     return result;
             }
@@ -64,15 +101,18 @@ public static class DataDictionaryExtensions
         }
 
         // Dictionary<string, object?> from ObjectToInferredTypesConverter
-        // Re-serialize to JSON then deserialize to target type with proper naming policy
+        // Re-serialize to JSON then deserialize to target type via ITextSerializer
         if (data is Dictionary<string, object?> dictionary)
         {
             try
             {
-                string dictJson = JsonSerializer.Serialize(dictionary, options);
-                var result = JsonSerializer.Deserialize<T>(dictJson, options);
-                if (result is not null)
-                    return result;
+                string? dictJson = serializer.SerializeToString(dictionary);
+                if (dictJson is not null)
+                {
+                    var result = serializer.Deserialize<T>(dictJson);
+                    if (result is not null)
+                        return result;
+                }
             }
             catch
             {
@@ -85,10 +125,13 @@ public static class DataDictionaryExtensions
         {
             try
             {
-                string listJson = JsonSerializer.Serialize(list, options);
-                var result = JsonSerializer.Deserialize<T>(listJson, options);
-                if (result is not null)
-                    return result;
+                string? listJson = serializer.SerializeToString(list);
+                if (listJson is not null)
+                {
+                    var result = serializer.Deserialize<T>(listJson);
+                    if (result is not null)
+                        return result;
+                }
             }
             catch
             {
@@ -111,12 +154,12 @@ public static class DataDictionaryExtensions
             }
         }
 
-        // JSON string
+        // JSON string - deserialize via ITextSerializer
         if (data is string json && json.IsJson())
         {
             try
             {
-                var result = JsonSerializer.Deserialize<T>(json, options);
+                var result = serializer.Deserialize<T>(json);
                 if (result is not null)
                     return result;
             }
@@ -142,49 +185,9 @@ public static class DataDictionaryExtensions
         return default;
     }
 
-    private static bool TryDeserialize<T>(JsonElement element, JsonSerializerOptions options, out T? result)
-    {
-        result = default;
-
-        try
-        {
-            // Fast-path for common primitives where the element isn't an object/array
-            // (Deserialize<T> also works for these, but this avoids some edge cases and allocations)
-            if (typeof(T) == typeof(string))
-            {
-                object? s = element.ValueKind switch
-                {
-                    JsonValueKind.String => element.GetString(),
-                    JsonValueKind.Number => element.GetRawText(),
-                    JsonValueKind.True => "true",
-                    JsonValueKind.False => "false",
-                    JsonValueKind.Null => null,
-                    _ => element.GetRawText()
-                };
-
-                result = (T?)s;
-                return true;
-            }
-
-            // General case
-            var deserialized = element.Deserialize<T>(options);
-            if (deserialized is not null)
-            {
-                result = deserialized;
-                return true;
-            }
-        }
-        catch
-        {
-            // Ignored
-        }
-
-        return false;
-    }
-
     public static void RemoveSensitiveData(this DataDictionary extendedData)
     {
-        string[] removeKeys = extendedData.Keys.Where(k => k.StartsWith('-')).ToArray();
+        string[] removeKeys = [.. extendedData.Keys.Where(k => k.StartsWith('-'))];
         foreach (string key in removeKeys)
             extendedData.Remove(key);
     }

src/Exceptionless.Core/Extensions/ErrorExtensions.cs

@@ -1,6 +1,6 @@
-using System.Text.Json;
 using Exceptionless.Core.Models;
 using Exceptionless.Core.Models.Data;
+using Foundatio.Serializer;
 
 namespace Exceptionless.Core.Extensions;
 
@@ -59,9 +59,9 @@ public static StackingTarget GetStackingTarget(this Error error)
         };
     }
 
-    public static StackingTarget? GetStackingTarget(this Event ev, JsonSerializerOptions options)
+    public static StackingTarget? GetStackingTarget(this Event ev, ITextSerializer serializer)
     {
-        var error = ev.GetError(options);
+        var error = ev.GetError(serializer);
         return error?.GetStackingTarget();
     }