diff --git a/src/EFCore.PG/Query/Internal/NpgsqlJsonPostprocessor.cs b/src/EFCore.PG/Query/Internal/NpgsqlJsonPostprocessor.cs new file mode 100644 index 000000000..300455a3b --- /dev/null +++ b/src/EFCore.PG/Query/Internal/NpgsqlJsonPostprocessor.cs @@ -0,0 +1,77 @@ +using Npgsql.EntityFrameworkCore.PostgreSQL.Query.Expressions.Internal; +using Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.Mapping; + +namespace Npgsql.EntityFrameworkCore.PostgreSQL.Query.Internal; + +/// +/// Applies JSON-related post-processing transformations. +/// +/// +/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to +/// the same compatibility standards as public APIs. It may be changed or removed without notice in +/// any release. You should only use it directly in your code with extreme caution and knowing that +/// doing so can result in application failures when updating to a new Entity Framework Core release. +/// +public class NpgsqlJsonPostprocessor(ISqlExpressionFactory sqlExpressionFactory) : ExpressionVisitor +{ + private readonly HashSet _jsonRecordsetAliases = []; + + /// + protected override Expression VisitExtension(Expression node) + { + switch (node) + { + case ShapedQueryExpression shapedQueryExpression: + return shapedQueryExpression.Update( + Visit(shapedQueryExpression.QueryExpression), + Visit(shapedQueryExpression.ShaperExpression)); + + case SelectExpression selectExpression: + // Binary values are stored as base64 strings in JSON, but json_to_recordset/jsonb_to_recordset do not decode + // them when extracting bytea columns. Find affected recordsets before visiting their column references. + foreach (var table in selectExpression.Tables) + { + if (table.UnwrapJoin() is PgTableValuedFunctionExpression + { + Name: "json_to_recordset" or "jsonb_to_recordset", + ColumnInfos: { } columnInfos + } tvf + && columnInfos.Any(c => c.TypeMapping is NpgsqlByteArrayTypeMapping)) + { + _jsonRecordsetAliases.Add(tvf.Alias); + } + } + + // EF assigns query-wide unique aliases, so keep these registered for correlated references and update setters. + return base.VisitExtension(node); + + case ColumnExpression { TypeMapping: NpgsqlByteArrayTypeMapping } column + when _jsonRecordsetAliases.Contains(column.TableAlias): + // Extract the base64 string as text and decode it, preserving the original binary result mapping. + // Rewrite every reference, not just projections, so predicates and other server-side operations use decoded bytes. + return sqlExpressionFactory.Function( + "decode", + [ + new ColumnExpression( + column.Name, column.TableAlias, typeof(string), NpgsqlStringTypeMapping.Default, column.IsNullable), + sqlExpressionFactory.Constant("base64") + ], + nullable: true, + argumentsPropagateNullability: [true, false], + column.Type, + column.TypeMapping); + + case PgTableValuedFunctionExpression { ColumnInfos: { } columnInfos } tvf + when _jsonRecordsetAliases.Contains(tvf.Alias): + // Change the recordset's column declarations to match the text inputs used by decode above. + return tvf.Update(this.Visit(tvf.Arguments)).WithColumnInfos( + columnInfos.Select( + c => c.TypeMapping is NpgsqlByteArrayTypeMapping + ? c with { TypeMapping = NpgsqlStringTypeMapping.Default } + : c).ToArray()); + + default: + return base.VisitExtension(node); + } + } +} diff --git a/src/EFCore.PG/Query/Internal/NpgsqlQueryTranslationPostprocessor.cs b/src/EFCore.PG/Query/Internal/NpgsqlQueryTranslationPostprocessor.cs index af6ebe647..72e948ec6 100644 --- a/src/EFCore.PG/Query/Internal/NpgsqlQueryTranslationPostprocessor.cs +++ b/src/EFCore.PG/Query/Internal/NpgsqlQueryTranslationPostprocessor.cs @@ -24,6 +24,8 @@ public override Expression Process(Expression query) { var result = base.Process(query); + // Structural projections require column expressions, so apply decoding after projection application. + result = new NpgsqlJsonPostprocessor(RelationalDependencies.SqlExpressionFactory).Visit(result); result = new NpgsqlUnnestPostprocessor().Visit(result); result = new NpgsqlSetOperationTypingInjector().Visit(result); result = new NpgsqlPeriodForeignKeyPostprocessor(RelationalQueryCompilationContext.QueryTrackingBehavior).Visit(result); diff --git a/test/EFCore.PG.FunctionalTests/Query/Translations/JsonTranslationsNpgsqlTest.cs b/test/EFCore.PG.FunctionalTests/Query/Translations/JsonTranslationsNpgsqlTest.cs index bbb2fb7e5..1f49eef00 100644 --- a/test/EFCore.PG.FunctionalTests/Query/Translations/JsonTranslationsNpgsqlTest.cs +++ b/test/EFCore.PG.FunctionalTests/Query/Translations/JsonTranslationsNpgsqlTest.cs @@ -137,6 +137,78 @@ LIMIT 2 }); } + [ConditionalTheory] + [InlineData(false, "Json")] + [InlineData(true, "Json")] + [InlineData(false, "Jsonb")] + [InlineData(true, "Jsonb")] + public virtual async Task Byte_array_in_json_collection(bool async, string propertyName) + { + await using var context = CreateContext(); + await context.Database.CreateExecutionStrategy().ExecuteAsync( + async () => + { + await using var transaction = await context.Database.BeginTransactionAsync(); + + List elements = + [ + new() { Name = "A", Content = [0, 1, 127, 128, 255] }, + new() { Name = "B", Content = [] }, + new() { Name = "C", Content = null } + ]; + context.Add( + new JsonByteArrayEntity + { + Id = 1, + Json = propertyName == "Json" ? elements : [], + Jsonb = propertyName == "Jsonb" ? elements : [] + }); + await context.SaveChangesAsync(); + context.ChangeTracker.Clear(); + + var entity = async + ? await context.Set().AsNoTracking().SingleAsync() + : context.Set().AsNoTracking().Single(); + var materializedElements = propertyName == "Json" ? entity.Json : entity.Jsonb; + Assert.Equal(elements.Select(e => e.Content), materializedElements.Select(e => e.Content)); + + Fixture.TestSqlLoggerFactory.Clear(); + + var collectionQuery = context.Set() + .SelectMany(e => EF.Property>(e, propertyName)); + var projectionQuery = collectionQuery.OrderBy(e => e.Name).Select(e => e.Content); + var projected = async ? await projectionQuery.ToListAsync() : projectionQuery.ToList(); + Assert.Equal(elements.Select(e => e.Content), projected); + + var predicateQuery = collectionQuery.Where(e => e.Content == new byte[] { 0, 1, 127, 128, 255 }).Select(e => e.Name); + var matchingName = async ? await predicateQuery.SingleAsync() : predicateQuery.Single(); + Assert.Equal("A", matchingName); + + var functionName = propertyName == "Json" ? "json_to_recordset" : "jsonb_to_recordset"; + AssertSql( + $""" +SELECT decode(j0."Content", 'base64') AS "Content" +FROM "JsonByteArrayEntity" AS j +JOIN LATERAL ROWS FROM ({functionName}(j."{propertyName}") AS ( + "Content" text, + "Name" text +)) WITH ORDINALITY AS j0 ON TRUE +ORDER BY j0."Name" NULLS FIRST +""", + // + $""" +SELECT j0."Name" +FROM "JsonByteArrayEntity" AS j +JOIN LATERAL ROWS FROM ({functionName}(j."{propertyName}") AS ( + "Content" text, + "Name" text +)) WITH ORDINALITY AS j0 ON TRUE +WHERE decode(j0."Content", 'base64') = BYTEA E'\\x00017F80FF' +LIMIT 2 +"""); + }); + } + public class ByteArrayJsonEntity { public int Id { get; set; } @@ -144,6 +216,19 @@ public class ByteArrayJsonEntity public byte[] Jsonb { get; set; } = null!; } + public class JsonByteArrayEntity + { + public int Id { get; set; } + public List Json { get; set; } = []; + public List Jsonb { get; set; } = []; + } + + public class ByteArrayJsonElement + { + public string Name { get; set; } = null!; + public byte[]? Content { get; set; } + } + public class JsonTranslationsQueryNpgsqlFixture : JsonTranslationsQueryFixtureBase, ITestSqlLoggerFactory { protected override ITestStoreFactory TestStoreFactory @@ -162,6 +247,13 @@ protected override void OnModelCreating(ModelBuilder modelBuilder, DbContext con b.Property(e => e.Json).HasColumnType("json"); b.Property(e => e.Jsonb).HasColumnType("jsonb"); }); + + modelBuilder.Entity(b => + { + b.Property(e => e.Id).ValueGeneratedNever(); + b.ComplexCollection(e => e.Json).ToJson().HasColumnType("json"); + b.ComplexCollection(e => e.Jsonb).ToJson().HasColumnType("jsonb"); + }); } protected override string RemoveJsonProperty(string column, string property)