From dea609545017e029a32094a9018cf96088a3a1f9 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Wed, 21 Oct 2026 17:10:57 +0200 Subject: [PATCH 1/3] Fix byte-array projections from JSON collections Decode base64 text extracted by json_to_recordset and jsonb_to_recordset before using binary properties in projections or predicates. Add sync/async regression coverage for json and jsonb, including null and empty payloads. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dbf67848-d1a3-45a1-a202-1b7161746a53 --- .../Query/Internal/NpgsqlJsonPostprocessor.cs | 71 ++++++++++++++ .../NpgsqlQueryTranslationPostprocessor.cs | 2 + .../JsonTranslationsNpgsqlTest.cs | 98 +++++++++++++++++++ 3 files changed, 171 insertions(+) create mode 100644 src/EFCore.PG/Query/Internal/NpgsqlJsonPostprocessor.cs diff --git a/src/EFCore.PG/Query/Internal/NpgsqlJsonPostprocessor.cs b/src/EFCore.PG/Query/Internal/NpgsqlJsonPostprocessor.cs new file mode 100644 index 000000000..58d417bae --- /dev/null +++ b/src/EFCore.PG/Query/Internal/NpgsqlJsonPostprocessor.cs @@ -0,0 +1,71 @@ +using Npgsql.EntityFrameworkCore.PostgreSQL.Query.Expressions.Internal; +using Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.Mapping; + +namespace Npgsql.EntityFrameworkCore.PostgreSQL.Query.Internal; + +/// +/// Applies base64 decoding to binary properties extracted by json_to_recordset/jsonb_to_recordset. +/// +/// +/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to +/// the same compatibility standards as public APIs. It may be changed or removed without notice in +/// any release. You should only use it directly in your code with extreme caution and knowing that +/// doing so can result in application failures when updating to a new Entity Framework Core release. +/// +public class NpgsqlJsonPostprocessor(ISqlExpressionFactory sqlExpressionFactory) : ExpressionVisitor +{ + private readonly HashSet _jsonRecordsetAliases = []; + + /// + protected override Expression VisitExtension(Expression node) + { + switch (node) + { + case ShapedQueryExpression shapedQueryExpression: + return shapedQueryExpression.Update( + Visit(shapedQueryExpression.QueryExpression), + Visit(shapedQueryExpression.ShaperExpression)); + + case SelectExpression selectExpression: + foreach (var table in selectExpression.Tables) + { + if (table.UnwrapJoin() is PgTableValuedFunctionExpression + { + Name: "json_to_recordset" or "jsonb_to_recordset", + ColumnInfos: { } columnInfos + } tvf + && columnInfos.Any(c => c.TypeMapping is NpgsqlByteArrayTypeMapping)) + { + _jsonRecordsetAliases.Add(tvf.Alias); + } + } + + return base.VisitExtension(node); + + case ColumnExpression { TypeMapping: NpgsqlByteArrayTypeMapping } column + when _jsonRecordsetAliases.Contains(column.TableAlias): + return sqlExpressionFactory.Function( + "decode", + [ + new ColumnExpression( + column.Name, column.TableAlias, typeof(string), NpgsqlStringTypeMapping.Default, column.IsNullable), + sqlExpressionFactory.Constant("base64") + ], + nullable: true, + argumentsPropagateNullability: [true, false], + column.Type, + column.TypeMapping); + + case PgTableValuedFunctionExpression { ColumnInfos: { } columnInfos } tvf + when _jsonRecordsetAliases.Contains(tvf.Alias): + return tvf.Update(this.Visit(tvf.Arguments)).WithColumnInfos( + columnInfos.Select( + c => c.TypeMapping is NpgsqlByteArrayTypeMapping + ? c with { TypeMapping = NpgsqlStringTypeMapping.Default } + : c).ToArray()); + + default: + return base.VisitExtension(node); + } + } +} diff --git a/src/EFCore.PG/Query/Internal/NpgsqlQueryTranslationPostprocessor.cs b/src/EFCore.PG/Query/Internal/NpgsqlQueryTranslationPostprocessor.cs index af6ebe647..72e948ec6 100644 --- a/src/EFCore.PG/Query/Internal/NpgsqlQueryTranslationPostprocessor.cs +++ b/src/EFCore.PG/Query/Internal/NpgsqlQueryTranslationPostprocessor.cs @@ -24,6 +24,8 @@ public override Expression Process(Expression query) { var result = base.Process(query); + // Structural projections require column expressions, so apply decoding after projection application. + result = new NpgsqlJsonPostprocessor(RelationalDependencies.SqlExpressionFactory).Visit(result); result = new NpgsqlUnnestPostprocessor().Visit(result); result = new NpgsqlSetOperationTypingInjector().Visit(result); result = new NpgsqlPeriodForeignKeyPostprocessor(RelationalQueryCompilationContext.QueryTrackingBehavior).Visit(result); diff --git a/test/EFCore.PG.FunctionalTests/Query/Translations/JsonTranslationsNpgsqlTest.cs b/test/EFCore.PG.FunctionalTests/Query/Translations/JsonTranslationsNpgsqlTest.cs index bbb2fb7e5..cc95c1d10 100644 --- a/test/EFCore.PG.FunctionalTests/Query/Translations/JsonTranslationsNpgsqlTest.cs +++ b/test/EFCore.PG.FunctionalTests/Query/Translations/JsonTranslationsNpgsqlTest.cs @@ -1,3 +1,5 @@ +#pragma warning disable EF8001 // ToJson on owned entities is obsolete + using System.Text.Json; namespace Microsoft.EntityFrameworkCore.Query.Translations; @@ -137,6 +139,78 @@ LIMIT 2 }); } + [ConditionalTheory] + [InlineData(false, "Json")] + [InlineData(true, "Json")] + [InlineData(false, "Jsonb")] + [InlineData(true, "Jsonb")] + public virtual async Task Byte_array_in_json_collection(bool async, string propertyName) + { + await using var context = CreateContext(); + await context.Database.CreateExecutionStrategy().ExecuteAsync( + async () => + { + await using var transaction = await context.Database.BeginTransactionAsync(); + + List elements = + [ + new() { Name = "A", Content = [0, 1, 127, 128, 255] }, + new() { Name = "B", Content = [] }, + new() { Name = "C", Content = null } + ]; + context.Add( + new JsonByteArrayEntity + { + Id = 1, + Json = propertyName == "Json" ? elements : [], + Jsonb = propertyName == "Jsonb" ? elements : [] + }); + await context.SaveChangesAsync(); + context.ChangeTracker.Clear(); + + var entity = async + ? await context.Set().AsNoTracking().SingleAsync() + : context.Set().AsNoTracking().Single(); + var materializedElements = propertyName == "Json" ? entity.Json : entity.Jsonb; + Assert.Equal(elements.Select(e => e.Content), materializedElements.Select(e => e.Content)); + + Fixture.TestSqlLoggerFactory.Clear(); + + var collectionQuery = context.Set() + .SelectMany(e => EF.Property>(e, propertyName)); + var projectionQuery = collectionQuery.OrderBy(e => e.Name).Select(e => e.Content); + var projected = async ? await projectionQuery.ToListAsync() : projectionQuery.ToList(); + Assert.Equal(elements.Select(e => e.Content), projected); + + var predicateQuery = collectionQuery.Where(e => e.Content == new byte[] { 0, 1, 127, 128, 255 }).Select(e => e.Name); + var matchingName = async ? await predicateQuery.SingleAsync() : predicateQuery.Single(); + Assert.Equal("A", matchingName); + + var functionName = propertyName == "Json" ? "json_to_recordset" : "jsonb_to_recordset"; + AssertSql( + $""" +SELECT decode(j0."Content", 'base64') AS "Content" +FROM "JsonByteArrayEntity" AS j +JOIN LATERAL ROWS FROM ({functionName}(j."{propertyName}") AS ( + "Content" text, + "Name" text +)) WITH ORDINALITY AS j0 ON TRUE +ORDER BY j0."Name" NULLS FIRST +""", + // + $""" +SELECT j0."Name" +FROM "JsonByteArrayEntity" AS j +JOIN LATERAL ROWS FROM ({functionName}(j."{propertyName}") AS ( + "Content" text, + "Name" text +)) WITH ORDINALITY AS j0 ON TRUE +WHERE decode(j0."Content", 'base64') = BYTEA E'\\x00017F80FF' +LIMIT 2 +"""); + }); + } + public class ByteArrayJsonEntity { public int Id { get; set; } @@ -144,11 +218,28 @@ public class ByteArrayJsonEntity public byte[] Jsonb { get; set; } = null!; } + public class JsonByteArrayEntity + { + public int Id { get; set; } + public List Json { get; set; } = []; + public List Jsonb { get; set; } = []; + } + + public class ByteArrayJsonElement + { + public string Name { get; set; } = null!; + public byte[]? Content { get; set; } + } + public class JsonTranslationsQueryNpgsqlFixture : JsonTranslationsQueryFixtureBase, ITestSqlLoggerFactory { protected override ITestStoreFactory TestStoreFactory => NpgsqlTestStoreFactory.Instance; + public override DbContextOptionsBuilder AddOptions(DbContextOptionsBuilder builder) + => base.AddOptions(builder) + .ConfigureWarnings(w => w.Ignore(RelationalEventId.OwnedEntityMappedToJsonCollectionWarning)); + protected override void OnModelCreating(ModelBuilder modelBuilder, DbContext context) { base.OnModelCreating(modelBuilder, context); @@ -162,6 +253,13 @@ protected override void OnModelCreating(ModelBuilder modelBuilder, DbContext con b.Property(e => e.Json).HasColumnType("json"); b.Property(e => e.Jsonb).HasColumnType("jsonb"); }); + + modelBuilder.Entity(b => + { + b.Property(e => e.Id).ValueGeneratedNever(); + b.OwnsMany(e => e.Json).ToJson().HasColumnType("json"); + b.OwnsMany(e => e.Jsonb).ToJson().HasColumnType("jsonb"); + }); } protected override string RemoveJsonProperty(string column, string property) From f6e1cd55502bdd3f7874193677857671eb064363 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Thu, 22 Oct 2026 14:06:59 +0200 Subject: [PATCH 2/3] Use complex JSON mapping in binary projection regression Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dbf67848-d1a3-45a1-a202-1b7161746a53 --- .../Query/Translations/JsonTranslationsNpgsqlTest.cs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/test/EFCore.PG.FunctionalTests/Query/Translations/JsonTranslationsNpgsqlTest.cs b/test/EFCore.PG.FunctionalTests/Query/Translations/JsonTranslationsNpgsqlTest.cs index cc95c1d10..1f49eef00 100644 --- a/test/EFCore.PG.FunctionalTests/Query/Translations/JsonTranslationsNpgsqlTest.cs +++ b/test/EFCore.PG.FunctionalTests/Query/Translations/JsonTranslationsNpgsqlTest.cs @@ -1,5 +1,3 @@ -#pragma warning disable EF8001 // ToJson on owned entities is obsolete - using System.Text.Json; namespace Microsoft.EntityFrameworkCore.Query.Translations; @@ -236,10 +234,6 @@ public class JsonTranslationsQueryNpgsqlFixture : JsonTranslationsQueryFixtureBa protected override ITestStoreFactory TestStoreFactory => NpgsqlTestStoreFactory.Instance; - public override DbContextOptionsBuilder AddOptions(DbContextOptionsBuilder builder) - => base.AddOptions(builder) - .ConfigureWarnings(w => w.Ignore(RelationalEventId.OwnedEntityMappedToJsonCollectionWarning)); - protected override void OnModelCreating(ModelBuilder modelBuilder, DbContext context) { base.OnModelCreating(modelBuilder, context); @@ -257,8 +251,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder, DbContext con modelBuilder.Entity(b => { b.Property(e => e.Id).ValueGeneratedNever(); - b.OwnsMany(e => e.Json).ToJson().HasColumnType("json"); - b.OwnsMany(e => e.Jsonb).ToJson().HasColumnType("jsonb"); + b.ComplexCollection(e => e.Json).ToJson().HasColumnType("json"); + b.ComplexCollection(e => e.Jsonb).ToJson().HasColumnType("jsonb"); }); } From 76115f86c08d8c3d0c02ddf4fbc502c95096c1c5 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Thu, 22 Oct 2026 14:27:01 +0200 Subject: [PATCH 3/3] Explain JSON postprocessing transformations Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dbf67848-d1a3-45a1-a202-1b7161746a53 --- src/EFCore.PG/Query/Internal/NpgsqlJsonPostprocessor.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/EFCore.PG/Query/Internal/NpgsqlJsonPostprocessor.cs b/src/EFCore.PG/Query/Internal/NpgsqlJsonPostprocessor.cs index 58d417bae..300455a3b 100644 --- a/src/EFCore.PG/Query/Internal/NpgsqlJsonPostprocessor.cs +++ b/src/EFCore.PG/Query/Internal/NpgsqlJsonPostprocessor.cs @@ -4,7 +4,7 @@ namespace Npgsql.EntityFrameworkCore.PostgreSQL.Query.Internal; /// -/// Applies base64 decoding to binary properties extracted by json_to_recordset/jsonb_to_recordset. +/// Applies JSON-related post-processing transformations. /// /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -27,6 +27,8 @@ protected override Expression VisitExtension(Expression node) Visit(shapedQueryExpression.ShaperExpression)); case SelectExpression selectExpression: + // Binary values are stored as base64 strings in JSON, but json_to_recordset/jsonb_to_recordset do not decode + // them when extracting bytea columns. Find affected recordsets before visiting their column references. foreach (var table in selectExpression.Tables) { if (table.UnwrapJoin() is PgTableValuedFunctionExpression @@ -40,10 +42,13 @@ protected override Expression VisitExtension(Expression node) } } + // EF assigns query-wide unique aliases, so keep these registered for correlated references and update setters. return base.VisitExtension(node); case ColumnExpression { TypeMapping: NpgsqlByteArrayTypeMapping } column when _jsonRecordsetAliases.Contains(column.TableAlias): + // Extract the base64 string as text and decode it, preserving the original binary result mapping. + // Rewrite every reference, not just projections, so predicates and other server-side operations use decoded bytes. return sqlExpressionFactory.Function( "decode", [ @@ -58,6 +63,7 @@ when _jsonRecordsetAliases.Contains(column.TableAlias): case PgTableValuedFunctionExpression { ColumnInfos: { } columnInfos } tvf when _jsonRecordsetAliases.Contains(tvf.Alias): + // Change the recordset's column declarations to match the text inputs used by decode above. return tvf.Update(this.Visit(tvf.Arguments)).WithColumnInfos( columnInfos.Select( c => c.TypeMapping is NpgsqlByteArrayTypeMapping