diff --git a/src/EFCore.PG/Query/Internal/NpgsqlJsonPostprocessor.cs b/src/EFCore.PG/Query/Internal/NpgsqlJsonPostprocessor.cs new file mode 100644 index 000000000..300455a3b --- /dev/null +++ b/src/EFCore.PG/Query/Internal/NpgsqlJsonPostprocessor.cs @@ -0,0 +1,77 @@ +using Npgsql.EntityFrameworkCore.PostgreSQL.Query.Expressions.Internal; +using Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.Mapping; + +namespace Npgsql.EntityFrameworkCore.PostgreSQL.Query.Internal; + +/// +/// Applies JSON-related post-processing transformations. +/// +/// +/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to +/// the same compatibility standards as public APIs. It may be changed or removed without notice in +/// any release. You should only use it directly in your code with extreme caution and knowing that +/// doing so can result in application failures when updating to a new Entity Framework Core release. +/// +public class NpgsqlJsonPostprocessor(ISqlExpressionFactory sqlExpressionFactory) : ExpressionVisitor +{ + private readonly HashSet _jsonRecordsetAliases = []; + + /// + protected override Expression VisitExtension(Expression node) + { + switch (node) + { + case ShapedQueryExpression shapedQueryExpression: + return shapedQueryExpression.Update( + Visit(shapedQueryExpression.QueryExpression), + Visit(shapedQueryExpression.ShaperExpression)); + + case SelectExpression selectExpression: + // Binary values are stored as base64 strings in JSON, but json_to_recordset/jsonb_to_recordset do not decode + // them when extracting bytea columns. Find affected recordsets before visiting their column references. + foreach (var table in selectExpression.Tables) + { + if (table.UnwrapJoin() is PgTableValuedFunctionExpression + { + Name: "json_to_recordset" or "jsonb_to_recordset", + ColumnInfos: { } columnInfos + } tvf + && columnInfos.Any(c => c.TypeMapping is NpgsqlByteArrayTypeMapping)) + { + _jsonRecordsetAliases.Add(tvf.Alias); + } + } + + // EF assigns query-wide unique aliases, so keep these registered for correlated references and update setters. + return base.VisitExtension(node); + + case ColumnExpression { TypeMapping: NpgsqlByteArrayTypeMapping } column + when _jsonRecordsetAliases.Contains(column.TableAlias): + // Extract the base64 string as text and decode it, preserving the original binary result mapping. + // Rewrite every reference, not just projections, so predicates and other server-side operations use decoded bytes. + return sqlExpressionFactory.Function( + "decode", + [ + new ColumnExpression( + column.Name, column.TableAlias, typeof(string), NpgsqlStringTypeMapping.Default, column.IsNullable), + sqlExpressionFactory.Constant("base64") + ], + nullable: true, + argumentsPropagateNullability: [true, false], + column.Type, + column.TypeMapping); + + case PgTableValuedFunctionExpression { ColumnInfos: { } columnInfos } tvf + when _jsonRecordsetAliases.Contains(tvf.Alias): + // Change the recordset's column declarations to match the text inputs used by decode above. + return tvf.Update(this.Visit(tvf.Arguments)).WithColumnInfos( + columnInfos.Select( + c => c.TypeMapping is NpgsqlByteArrayTypeMapping + ? c with { TypeMapping = NpgsqlStringTypeMapping.Default } + : c).ToArray()); + + default: + return base.VisitExtension(node); + } + } +} diff --git a/src/EFCore.PG/Query/Internal/NpgsqlQueryTranslationPostprocessor.cs b/src/EFCore.PG/Query/Internal/NpgsqlQueryTranslationPostprocessor.cs index 0bb5e96d6..26eb16d8e 100644 --- a/src/EFCore.PG/Query/Internal/NpgsqlQueryTranslationPostprocessor.cs +++ b/src/EFCore.PG/Query/Internal/NpgsqlQueryTranslationPostprocessor.cs @@ -34,6 +34,8 @@ public override Expression Process(Expression query) { var result = base.Process(query); + // Structural projections require column expressions, so apply decoding after projection application. + result = new NpgsqlJsonPostprocessor(RelationalDependencies.SqlExpressionFactory).Visit(result); result = new NpgsqlUnnestPostprocessor().Visit(result); result = new NpgsqlSetOperationTypingInjector().Visit(result); diff --git a/test/EFCore.PG.FunctionalTests/Query/AdHocJsonQueryNpgsqlTest.cs b/test/EFCore.PG.FunctionalTests/Query/AdHocJsonQueryNpgsqlTest.cs index db1c9da08..426031b4d 100644 --- a/test/EFCore.PG.FunctionalTests/Query/AdHocJsonQueryNpgsqlTest.cs +++ b/test/EFCore.PG.FunctionalTests/Query/AdHocJsonQueryNpgsqlTest.cs @@ -206,6 +206,86 @@ LIMIT 2 } } + [ConditionalTheory] + [InlineData(false, "json")] + [InlineData(true, "json")] + [InlineData(false, "jsonb")] + [InlineData(true, "jsonb")] + public virtual async Task Byte_array_in_json_collection(bool async, string storeType) + { + List elements = + [ + new() { Name = "A", Content = [0, 1, 127, 128, 255] }, + new() { Name = "B", Content = [] }, + new() { Name = "C", Content = null } + ]; + var contextFactory = await InitializeAsync( + onModelCreating: modelBuilder => modelBuilder.Entity() + .ComplexCollection(e => e.Json).ToJson().HasColumnType(storeType), + seed: async context => + { + context.Entities.Add(new ByteArrayJsonContext.Entity { Json = elements }); + await context.SaveChangesAsync(); + }); + + await using var context = contextFactory.CreateContext(); + var entity = async + ? await context.Entities.AsNoTracking().SingleAsync() + : context.Entities.AsNoTracking().Single(); + Assert.Equal(elements.Select(e => e.Content), entity.Json.Select(e => e.Content)); + + TestSqlLoggerFactory.Clear(); + + var collectionQuery = context.Entities.SelectMany(e => e.Json); + var projectionQuery = collectionQuery.OrderBy(e => e.Name).Select(e => e.Content); + var projected = async ? await projectionQuery.ToListAsync() : projectionQuery.ToList(); + Assert.Equal(elements.Select(e => e.Content), projected); + + var predicateQuery = collectionQuery.Where(e => e.Content == new byte[] { 0, 1, 127, 128, 255 }).Select(e => e.Name); + var matchingName = async ? await predicateQuery.SingleAsync() : predicateQuery.Single(); + Assert.Equal("A", matchingName); + + var functionName = storeType == "json" ? "json_to_recordset" : "jsonb_to_recordset"; + AssertSql( + $""" +SELECT decode(j."Content", 'base64') AS "Content" +FROM "Entities" AS e +JOIN LATERAL ROWS FROM ({functionName}(e."Json") AS ( + "Content" text, + "Name" text +)) WITH ORDINALITY AS j ON TRUE +ORDER BY j."Name" NULLS FIRST +""", + // + $""" +SELECT j."Name" +FROM "Entities" AS e +JOIN LATERAL ROWS FROM ({functionName}(e."Json") AS ( + "Content" text, + "Name" text +)) WITH ORDINALITY AS j ON TRUE +WHERE decode(j."Content", 'base64') = BYTEA E'\\x00017F80FF' +LIMIT 2 +"""); + } + + protected class ByteArrayJsonContext(DbContextOptions options) : DbContext(options) + { + public DbSet Entities => Set(); + + public class Entity + { + public int Id { get; set; } + public List Json { get; set; } = []; + } + + public class JsonElement + { + public string Name { get; set; } + public byte[] Content { get; set; } + } + } + [ConditionalTheory, MemberData(nameof(IsAsyncData))] public virtual async Task Json_predicate_on_interval(bool async) {