Fix byte-array projections from JSON collections

src/EFCore.PG/Query/Internal/NpgsqlJsonPostprocessor.cs

@@ -0,0 +1,77 @@
+using Npgsql.EntityFrameworkCore.PostgreSQL.Query.Expressions.Internal;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.Mapping;
+
+namespace Npgsql.EntityFrameworkCore.PostgreSQL.Query.Internal;
+
+/// <summary>
+///     Applies JSON-related post-processing transformations.
+/// </summary>
+/// <remarks>
+///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+///     any release. You should only use it directly in your code with extreme caution and knowing that
+///     doing so can result in application failures when updating to a new Entity Framework Core release.
+/// </remarks>
+public class NpgsqlJsonPostprocessor(ISqlExpressionFactory sqlExpressionFactory) : ExpressionVisitor
+{
+    private readonly HashSet<string> _jsonRecordsetAliases = [];
+
+    /// <inheritdoc />
+    protected override Expression VisitExtension(Expression node)
+    {
+        switch (node)
+        {
+            case ShapedQueryExpression shapedQueryExpression:
+                return shapedQueryExpression.Update(
+                    Visit(shapedQueryExpression.QueryExpression),
+                    Visit(shapedQueryExpression.ShaperExpression));
+
+            case SelectExpression selectExpression:
+                // Binary values are stored as base64 strings in JSON, but json_to_recordset/jsonb_to_recordset do not decode
+                // them when extracting bytea columns. Find affected recordsets before visiting their column references.
+                foreach (var table in selectExpression.Tables)
+                {
+                    if (table.UnwrapJoin() is PgTableValuedFunctionExpression
+                        {
+                            Name: "json_to_recordset" or "jsonb_to_recordset",
+                            ColumnInfos: { } columnInfos
+                        } tvf
+                        && columnInfos.Any(c => c.TypeMapping is NpgsqlByteArrayTypeMapping))
+                    {
+                        _jsonRecordsetAliases.Add(tvf.Alias);
+                    }
+                }
+
+                // EF assigns query-wide unique aliases, so keep these registered for correlated references and update setters.
+                return base.VisitExtension(node);
+
+            case ColumnExpression { TypeMapping: NpgsqlByteArrayTypeMapping } column
+                when _jsonRecordsetAliases.Contains(column.TableAlias):
+                // Extract the base64 string as text and decode it, preserving the original binary result mapping.
+                // Rewrite every reference, not just projections, so predicates and other server-side operations use decoded bytes.
+                return sqlExpressionFactory.Function(
+                    "decode",
+                    [
+                        new ColumnExpression(
+                            column.Name, column.TableAlias, typeof(string), NpgsqlStringTypeMapping.Default, column.IsNullable),
+                        sqlExpressionFactory.Constant("base64")
+                    ],
+                    nullable: true,
+                    argumentsPropagateNullability: [true, false],
+                    column.Type,
+                    column.TypeMapping);
+
+            case PgTableValuedFunctionExpression { ColumnInfos: { } columnInfos } tvf
+                when _jsonRecordsetAliases.Contains(tvf.Alias):
+                // Change the recordset's column declarations to match the text inputs used by decode above.
+                return tvf.Update(this.Visit(tvf.Arguments)).WithColumnInfos(
+                    columnInfos.Select(
+                        c => c.TypeMapping is NpgsqlByteArrayTypeMapping
+                            ? c with { TypeMapping = NpgsqlStringTypeMapping.Default }
+                            : c).ToArray());
+
+            default:
+                return base.VisitExtension(node);
+        }
+    }
+}

src/EFCore.PG/Query/Internal/NpgsqlQueryTranslationPostprocessor.cs

@@ -24,6 +24,8 @@ public override Expression Process(Expression query)
     {
         var result = base.Process(query);
 
+        // Structural projections require column expressions, so apply decoding after projection application.
+        result = new NpgsqlJsonPostprocessor(RelationalDependencies.SqlExpressionFactory).Visit(result);
         result = new NpgsqlUnnestPostprocessor().Visit(result);
         result = new NpgsqlSetOperationTypingInjector().Visit(result);
         result = new NpgsqlPeriodForeignKeyPostprocessor(RelationalQueryCompilationContext.QueryTrackingBehavior).Visit(result);

test/EFCore.PG.FunctionalTests/Query/Translations/JsonTranslationsNpgsqlTest.cs

@@ -137,13 +137,98 @@ LIMIT 2
             });
     }
 
+    [ConditionalTheory]
+    [InlineData(false, "Json")]
+    [InlineData(true, "Json")]
+    [InlineData(false, "Jsonb")]
+    [InlineData(true, "Jsonb")]
+    public virtual async Task Byte_array_in_json_collection(bool async, string propertyName)
+    {
+        await using var context = CreateContext();
+        await context.Database.CreateExecutionStrategy().ExecuteAsync(
+            async () =>
+            {
+                await using var transaction = await context.Database.BeginTransactionAsync();
+
+                List<ByteArrayJsonElement> elements =
+                [
+                    new() { Name = "A", Content = [0, 1, 127, 128, 255] },
+                    new() { Name = "B", Content = [] },
+                    new() { Name = "C", Content = null }
+                ];
+                context.Add(
+                    new JsonByteArrayEntity
+                    {
+                        Id = 1,
+                        Json = propertyName == "Json" ? elements : [],
+                        Jsonb = propertyName == "Jsonb" ? elements : []
+                    });
+                await context.SaveChangesAsync();
+                context.ChangeTracker.Clear();
+
+                var entity = async
+                    ? await context.Set<JsonByteArrayEntity>().AsNoTracking().SingleAsync()
+                    : context.Set<JsonByteArrayEntity>().AsNoTracking().Single();
+                var materializedElements = propertyName == "Json" ? entity.Json : entity.Jsonb;
+                Assert.Equal(elements.Select(e => e.Content), materializedElements.Select(e => e.Content));
+
+                Fixture.TestSqlLoggerFactory.Clear();
+
+                var collectionQuery = context.Set<JsonByteArrayEntity>()
+                    .SelectMany(e => EF.Property<List<ByteArrayJsonElement>>(e, propertyName));
+                var projectionQuery = collectionQuery.OrderBy(e => e.Name).Select(e => e.Content);
+                var projected = async ? await projectionQuery.ToListAsync() : projectionQuery.ToList();
+                Assert.Equal(elements.Select(e => e.Content), projected);
+
+                var predicateQuery = collectionQuery.Where(e => e.Content == new byte[] { 0, 1, 127, 128, 255 }).Select(e => e.Name);
+                var matchingName = async ? await predicateQuery.SingleAsync() : predicateQuery.Single();
+                Assert.Equal("A", matchingName);
+
+                var functionName = propertyName == "Json" ? "json_to_recordset" : "jsonb_to_recordset";
+                AssertSql(
+                    $"""
+SELECT decode(j0."Content", 'base64') AS "Content"
+FROM "JsonByteArrayEntity" AS j
+JOIN LATERAL ROWS FROM ({functionName}(j."{propertyName}") AS (
+    "Content" text,
+    "Name" text
+)) WITH ORDINALITY AS j0 ON TRUE
+ORDER BY j0."Name" NULLS FIRST
+""",
+                    //
+                    $"""
+SELECT j0."Name"
+FROM "JsonByteArrayEntity" AS j
+JOIN LATERAL ROWS FROM ({functionName}(j."{propertyName}") AS (
+    "Content" text,
+    "Name" text
+)) WITH ORDINALITY AS j0 ON TRUE
+WHERE decode(j0."Content", 'base64') = BYTEA E'\\x00017F80FF'
+LIMIT 2
+""");
+            });
+    }
+
     public class ByteArrayJsonEntity
     {
         public int Id { get; set; }
         public byte[] Json { get; set; } = null!;
         public byte[] Jsonb { get; set; } = null!;
     }
 
+    public class JsonByteArrayEntity
+    {
+        public int Id { get; set; }
+        public List<ByteArrayJsonElement> Json { get; set; } = [];
+        public List<ByteArrayJsonElement> Jsonb { get; set; } = [];
+    }
+
+    public class ByteArrayJsonElement
+    {
+        public string Name { get; set; } = null!;
+        public byte[]? Content { get; set; }
+    }
+
     public class JsonTranslationsQueryNpgsqlFixture : JsonTranslationsQueryFixtureBase, ITestSqlLoggerFactory
     {
         protected override ITestStoreFactory TestStoreFactory
@@ -162,6 +247,13 @@ protected override void OnModelCreating(ModelBuilder modelBuilder, DbContext con
                 b.Property(e => e.Json).HasColumnType("json");
                 b.Property(e => e.Jsonb).HasColumnType("jsonb");
             });
+
+            modelBuilder.Entity<JsonByteArrayEntity>(b =>
+            {
+                b.Property(e => e.Id).ValueGeneratedNever();
+                b.ComplexCollection(e => e.Json).ToJson().HasColumnType("json");
+                b.ComplexCollection(e => e.Jsonb).ToJson().HasColumnType("jsonb");
+            });
         }
 
         protected override string RemoveJsonProperty(string column, string property)