Fix ValuesExpression pruning dropping outer columns referenced from VALUES cells

src/EFCore.Relational/Query/SqlTreePruner.cs

@@ -273,10 +273,17 @@ protected virtual SelectExpression PruneSelect(SelectExpression select, bool pre
     ///     that ordering isn't actually necessary.
     /// </summary>
     /// <remarks>
-    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
-    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
-    ///     any release. You should only use it directly in your code with extreme caution and knowing that
-    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    ///     <para>
+    ///         This also visits the row/parameter contents of the <see cref="ValuesExpression" />, so that column
+    ///         references embedded within it (e.g. a navigation column inlined into a row) get registered in
+    ///         <see cref="ReferencedColumnMap" /> before the tables providing them are considered for pruning.
+    ///     </para>
+    ///     <para>
+    ///         This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///         the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///         any release. You should only use it directly in your code with extreme caution and knowing that
+    ///         doing so can result in application failures when updating to a new Entity Framework Core release.
+    ///     </para>
     /// </remarks>
     [EntityFrameworkInternal]
     protected virtual ValuesExpression PruneValues(ValuesExpression values)
@@ -301,7 +308,7 @@ protected virtual ValuesExpression PruneValues(ValuesExpression values)
                     for (var j = 0; j < i; j++)
                     {
                         referencedColumns[j] = true;
-                        newColumnNames.Add(columnName);
+                        newColumnNames.Add(values.ColumnNames[j]);
                     }
                 }
 
@@ -337,24 +344,32 @@ protected virtual ValuesExpression PruneValues(ValuesExpression values)
             }
         }
 
-        if (referencedColumns is null)
-        {
-            return values;
-        }
-
-        // We know at least some columns are getting pruned.
-        Debug.Assert(newColumnNames is not null);
-
+        // Always visit nested expressions so that column references inside VALUES cells (e.g. navigation
+        // columns embedded in an inline collection) are registered before outer tables are pruned (#38700).
         switch (values)
         {
             // If we have a value parameter (row values aren't specific in line), we still prune the column names.
             // Later in SqlNullabilityProcessor, when the parameterized collection is inline to constants, we'll take
             // the column names into account.
             case { ValuesParameter: not null }:
-                return new ValuesExpression(values.Alias, rowValues: null, values.ValuesParameter, newColumnNames);
+            {
+                var visitedParameter = (SqlParameterExpression)Visit(values.ValuesParameter);
+
+                return referencedColumns is null
+                    ? values.Update(visitedParameter)
+                    : new ValuesExpression(values.Alias, rowValues: null, visitedParameter, newColumnNames!);
+            }
 
             // Go over the rows and create new ones without the pruned columns.
             case { RowValues: { } rowValues }:
+            {
+                if (referencedColumns is null)
+                {
+                    return values.Update(this.VisitAndConvert(rowValues));
+                }
+
+                Debug.Assert(newColumnNames is not null);
+
                 var newRowValues = new RowValueExpression[rowValues.Count];
 
                 for (var i = 0; i < rowValues.Count; i++)
@@ -366,14 +381,15 @@ protected virtual ValuesExpression PruneValues(ValuesExpression values)
                     {
                         if (referencedColumns[j])
                         {
-                            newValues.Add(oldValues[j]);
+                            newValues.Add((SqlExpression)Visit(oldValues[j]));
                         }
                     }
 
                     newRowValues[i] = new RowValueExpression(newValues);
                 }
 
                 return new ValuesExpression(values.Alias, newRowValues, valuesParameter: null, newColumnNames);
+            }
 
             default:
                 throw new UnreachableException();

test/EFCore.Relational.Specification.Tests/Query/AdHocQueryFiltersQueryRelationalTestBase.cs

@@ -15,4 +15,68 @@ protected void ClearLog()
 
     protected void AssertSql(params string[] expected)
         => TestSqlLoggerFactory.AssertBaseline(expected);
+
+    #region 38700
+
+    [Theory, MemberData(nameof(IsAsyncData))]
+    public virtual async Task Query_filter_with_inline_collection_of_navigation_column(bool async)
+    {
+        var contextFactory = await InitializeNonSharedTest<Context38700>(seed: c => c.SeedAsync());
+        using var context = contextFactory.CreateDbContext();
+
+        Context38700.AuthorizedServiceIds = [10];
+
+        var query = context.Children.AsNoTracking().Select(c => c.Label);
+
+        var results = async
+            ? await query.ToListAsync()
+            : query.ToList();
+
+        Assert.Equal(["ok"], results);
+    }
+
+    protected class Context38700(DbContextOptions options) : DbContext(options)
+    {
+        public static List<int> AuthorizedServiceIds { get; set; } = [];
+
+        public DbSet<Parent38700> Parents
+            => Set<Parent38700>();
+
+        public DbSet<Child38700> Children
+            => Set<Child38700>();
+
+        protected override void OnModelCreating(ModelBuilder modelBuilder)
+        {
+            modelBuilder.Entity<Parent38700>().HasQueryFilter(
+                p => AuthorizedServiceIds.Contains(p.ServiceId));
+
+            // Inline array of a navigation column — triggers VALUES pruning of outer join columns (#38700).
+            modelBuilder.Entity<Child38700>().HasQueryFilter(c =>
+                new int?[] { c.Parent.ServiceId }
+                    .Any(id => id.HasValue && AuthorizedServiceIds.Contains(id.Value)));
+        }
+
+        public Task SeedAsync()
+        {
+            var parent = new Parent38700 { ServiceId = 10 };
+            Children.Add(new Child38700 { Parent = parent, Label = "ok" });
+            return SaveChangesAsync();
+        }
+    }
+
+    protected class Parent38700
+    {
+        public int Id { get; set; }
+        public int ServiceId { get; set; }
+    }
+
+    protected class Child38700
+    {
+        public int Id { get; set; }
+        public int ParentId { get; set; }
+        public Parent38700 Parent { get; set; }
+        public string Label { get; set; }
+    }
+
+    #endregion
 }

test/EFCore.Relational.Tests/Query/SqlTreePrunerTest.cs

@@ -0,0 +1,55 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Microsoft.EntityFrameworkCore.Query.SqlExpressions;
+
+namespace Microsoft.EntityFrameworkCore.Query;
+
+public class SqlTreePrunerTest
+{
+    [Fact]
+    public void PruneValues_preserves_leading_column_names_when_pruning_a_later_column()
+    {
+        // VALUES (_ord, Kept, Dropped) where only _ord and Kept are referenced.
+        // The first unreferenced column is at index 2, so PruneValues must backfill
+        // ColumnNames[0] and ColumnNames[1] — not repeat the pruned column's name (#38700 Copilot).
+        const string alias = "v";
+        var intMapping = IntTypeMapping.Default;
+
+        var values = new ValuesExpression(
+            alias,
+            [
+                new RowValueExpression(
+                [
+                    new SqlConstantExpression(0, intMapping),
+                    new SqlConstantExpression(10, intMapping),
+                    new SqlConstantExpression(20, intMapping)
+                ])
+            ],
+            [
+                RelationalQueryableMethodTranslatingExpressionVisitor.ValuesOrderingColumnName,
+                "Kept",
+                "Dropped"
+            ]);
+
+        var pruner = new TestSqlTreePruner();
+        pruner.RegisterColumn(alias, RelationalQueryableMethodTranslatingExpressionVisitor.ValuesOrderingColumnName);
+        pruner.RegisterColumn(alias, "Kept");
+
+        var pruned = pruner.PruneValuesPublic(values);
+
+        Assert.Equal(
+            [RelationalQueryableMethodTranslatingExpressionVisitor.ValuesOrderingColumnName, "Kept"],
+            pruned.ColumnNames);
+        Assert.Equal(2, Assert.Single(pruned.RowValues!).Values.Count);
+    }
+
+    private sealed class TestSqlTreePruner : SqlTreePruner
+    {
+        public void RegisterColumn(string tableAlias, string columnName)
+            => Visit(new ColumnExpression(columnName, tableAlias, typeof(int), IntTypeMapping.Default, nullable: false));
+
+        public ValuesExpression PruneValuesPublic(ValuesExpression values)
+            => PruneValues(values);
+    }
+}

test/EFCore.SqlServer.FunctionalTests/Query/AdHocQueryFiltersQuerySqlServerTest.cs

@@ -522,6 +522,26 @@ public override async Task Query_filter_with_EF_Parameter_throws()
         AssertSql();
     }
 
+    public override async Task Query_filter_with_inline_collection_of_navigation_column(bool async)
+    {
+        await base.Query_filter_with_inline_collection_of_navigation_column(async);
+
+        AssertSql(
+            """
+SELECT [c].[Label]
+FROM [Children] AS [c]
+INNER JOIN (
+    SELECT [p].[Id], [p].[ServiceId]
+    FROM [Parents] AS [p]
+    WHERE [p].[ServiceId] = 10
+) AS [p0] ON [c].[ParentId] = [p0].[Id]
+WHERE EXISTS (
+    SELECT 1
+    FROM (VALUES ([p0].[ServiceId])) AS [v]([Value])
+    WHERE [v].[Value] = 10)
+""");
+    }
+
     [Fact]
     public virtual void Check_all_tests_overridden()
         => TestHelpers.AssertAllMethodsOverridden(GetType());