[10.0] Fix byte-array projections from JSON collections

src/EFCore.PG/Query/Internal/NpgsqlJsonPostprocessor.cs

@@ -0,0 +1,77 @@
+using Npgsql.EntityFrameworkCore.PostgreSQL.Query.Expressions.Internal;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.Mapping;
+
+namespace Npgsql.EntityFrameworkCore.PostgreSQL.Query.Internal;
+
+/// <summary>
+///     Applies JSON-related post-processing transformations.
+/// </summary>
+/// <remarks>
+///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+///     any release. You should only use it directly in your code with extreme caution and knowing that
+///     doing so can result in application failures when updating to a new Entity Framework Core release.
+/// </remarks>
+public class NpgsqlJsonPostprocessor(ISqlExpressionFactory sqlExpressionFactory) : ExpressionVisitor
+{
+    private readonly HashSet<string> _jsonRecordsetAliases = [];
+
+    /// <inheritdoc />
+    protected override Expression VisitExtension(Expression node)
+    {
+        switch (node)
+        {
+            case ShapedQueryExpression shapedQueryExpression:
+                return shapedQueryExpression.Update(
+                    Visit(shapedQueryExpression.QueryExpression),
+                    Visit(shapedQueryExpression.ShaperExpression));
+
+            case SelectExpression selectExpression:
+                // Binary values are stored as base64 strings in JSON, but json_to_recordset/jsonb_to_recordset do not decode
+                // them when extracting bytea columns. Find affected recordsets before visiting their column references.
+                foreach (var table in selectExpression.Tables)
+                {
+                    if (table.UnwrapJoin() is PgTableValuedFunctionExpression
+                        {
+                            Name: "json_to_recordset" or "jsonb_to_recordset",
+                            ColumnInfos: { } columnInfos
+                        } tvf
+                        && columnInfos.Any(c => c.TypeMapping is NpgsqlByteArrayTypeMapping))
+                    {
+                        _jsonRecordsetAliases.Add(tvf.Alias);
+                    }
+                }
+
+                // EF assigns query-wide unique aliases, so keep these registered for correlated references and update setters.
+                return base.VisitExtension(node);
+
+            case ColumnExpression { TypeMapping: NpgsqlByteArrayTypeMapping } column
+                when _jsonRecordsetAliases.Contains(column.TableAlias):
+                // Extract the base64 string as text and decode it, preserving the original binary result mapping.
+                // Rewrite every reference, not just projections, so predicates and other server-side operations use decoded bytes.
+                return sqlExpressionFactory.Function(
+                    "decode",
+                    [
+                        new ColumnExpression(
+                            column.Name, column.TableAlias, typeof(string), NpgsqlStringTypeMapping.Default, column.IsNullable),
+                        sqlExpressionFactory.Constant("base64")
+                    ],
+                    nullable: true,
+                    argumentsPropagateNullability: [true, false],
+                    column.Type,
+                    column.TypeMapping);
+
+            case PgTableValuedFunctionExpression { ColumnInfos: { } columnInfos } tvf
+                when _jsonRecordsetAliases.Contains(tvf.Alias):
+                // Change the recordset's column declarations to match the text inputs used by decode above.
+                return tvf.Update(this.Visit(tvf.Arguments)).WithColumnInfos(
+                    columnInfos.Select(
+                        c => c.TypeMapping is NpgsqlByteArrayTypeMapping
+                            ? c with { TypeMapping = NpgsqlStringTypeMapping.Default }
+                            : c).ToArray());
+
+            default:
+                return base.VisitExtension(node);
+        }
+    }
+}

src/EFCore.PG/Query/Internal/NpgsqlQueryTranslationPostprocessor.cs

@@ -34,6 +34,8 @@ public override Expression Process(Expression query)
     {
         var result = base.Process(query);
 
+        // Structural projections require column expressions, so apply decoding after projection application.
+        result = new NpgsqlJsonPostprocessor(RelationalDependencies.SqlExpressionFactory).Visit(result);
         result = new NpgsqlUnnestPostprocessor().Visit(result);
         result = new NpgsqlSetOperationTypingInjector().Visit(result);
 

test/EFCore.PG.FunctionalTests/Query/AdHocJsonQueryNpgsqlTest.cs

@@ -206,6 +206,86 @@ LIMIT 2
         }
     }
 
+    [ConditionalTheory]
+    [InlineData(false, "json")]
+    [InlineData(true, "json")]
+    [InlineData(false, "jsonb")]
+    [InlineData(true, "jsonb")]
+    public virtual async Task Byte_array_in_json_collection(bool async, string storeType)
+    {
+        List<ByteArrayJsonContext.JsonElement> elements =
+        [
+            new() { Name = "A", Content = [0, 1, 127, 128, 255] },
+            new() { Name = "B", Content = [] },
+            new() { Name = "C", Content = null }
+        ];
+        var contextFactory = await InitializeAsync<ByteArrayJsonContext>(
+            onModelCreating: modelBuilder => modelBuilder.Entity<ByteArrayJsonContext.Entity>()
+                .ComplexCollection(e => e.Json).ToJson().HasColumnType(storeType),
+            seed: async context =>
+            {
+                context.Entities.Add(new ByteArrayJsonContext.Entity { Json = elements });
+                await context.SaveChangesAsync();
+            });
+
+        await using var context = contextFactory.CreateContext();
+        var entity = async
+            ? await context.Entities.AsNoTracking().SingleAsync()
+            : context.Entities.AsNoTracking().Single();
+        Assert.Equal(elements.Select(e => e.Content), entity.Json.Select(e => e.Content));
+
+        TestSqlLoggerFactory.Clear();
+
+        var collectionQuery = context.Entities.SelectMany(e => e.Json);
+        var projectionQuery = collectionQuery.OrderBy(e => e.Name).Select(e => e.Content);
+        var projected = async ? await projectionQuery.ToListAsync() : projectionQuery.ToList();
+        Assert.Equal(elements.Select(e => e.Content), projected);
+
+        var predicateQuery = collectionQuery.Where(e => e.Content == new byte[] { 0, 1, 127, 128, 255 }).Select(e => e.Name);
+        var matchingName = async ? await predicateQuery.SingleAsync() : predicateQuery.Single();
+        Assert.Equal("A", matchingName);
+
+        var functionName = storeType == "json" ? "json_to_recordset" : "jsonb_to_recordset";
+        AssertSql(
+            $"""
+SELECT decode(j."Content", 'base64') AS "Content"
+FROM "Entities" AS e
+JOIN LATERAL ROWS FROM ({functionName}(e."Json") AS (
+    "Content" text,
+    "Name" text
+)) WITH ORDINALITY AS j ON TRUE
+ORDER BY j."Name" NULLS FIRST
+""",
+            //
+            $"""
+SELECT j."Name"
+FROM "Entities" AS e
+JOIN LATERAL ROWS FROM ({functionName}(e."Json") AS (
+    "Content" text,
+    "Name" text
+)) WITH ORDINALITY AS j ON TRUE
+WHERE decode(j."Content", 'base64') = BYTEA E'\\x00017F80FF'
+LIMIT 2
+""");
+    }
+
+    protected class ByteArrayJsonContext(DbContextOptions options) : DbContext(options)
+    {
+        public DbSet<Entity> Entities => Set<Entity>();
+
+        public class Entity
+        {
+            public int Id { get; set; }
+            public List<JsonElement> Json { get; set; } = [];
+        }
+
+        public class JsonElement
+        {
+            public string Name { get; set; }
+            public byte[] Content { get; set; }
+        }
+    }
+
     [ConditionalTheory, MemberData(nameof(IsAsyncData))]
     public virtual async Task Json_predicate_on_interval(bool async)
     {