From 0a7b4c7821e2e2e6a4db0a216261aee51946d22c Mon Sep 17 00:00:00 2001 From: Emil Redzik Date: Wed, 25 Mar 2026 13:42:51 +0100 Subject: [PATCH 1/4] feat(orm): enhance lateral join dialect with orderBy support for array aggregation - Updated `buildArrayAgg` method to accept an optional `orderBy` parameter for sorting. - Introduced `buildRelationOrderByExpressions` to handle orderBy logic for relations. - Added tests for stable ordering in nested includes and selects for both SQLite and PostgreSQL. --- .../dialects/lateral-join-dialect-base.ts | 41 ++++- .../src/client/crud/dialects/postgresql.ts | 22 ++- .../relation/order-by-nested-includes.test.ts | 156 ++++++++++++++++++ 3 files changed, 213 insertions(+), 6 deletions(-) create mode 100644 tests/e2e/orm/client-api/relation/order-by-nested-includes.test.ts diff --git a/packages/orm/src/client/crud/dialects/lateral-join-dialect-base.ts b/packages/orm/src/client/crud/dialects/lateral-join-dialect-base.ts index cbcdbee30..ed2397795 100644 --- a/packages/orm/src/client/crud/dialects/lateral-join-dialect-base.ts +++ b/packages/orm/src/client/crud/dialects/lateral-join-dialect-base.ts @@ -2,9 +2,10 @@ import { invariant } from '@zenstackhq/common-helpers'; import { type AliasableExpression, type Expression, type ExpressionBuilder, type SelectQueryBuilder } from 'kysely'; import type { FieldDef, GetModels, SchemaDef } from '../../../schema'; import { DELEGATE_JOINED_FIELD_PREFIX } from '../../constants'; -import type { FindArgs } from '../../crud-types'; +import type { FindArgs, NullsOrder, SortOrder } from '../../crud-types'; import { buildJoinPairs, + ensureArray, getDelegateDescendantModels, getManyToManyRelation, isRelationField, @@ -23,7 +24,10 @@ export abstract class LateralJoinDialectBase extends B /** * Builds an array aggregation expression. */ - protected abstract buildArrayAgg(arg: Expression): AliasableExpression; + protected abstract buildArrayAgg( + arg: Expression, + orderBy?: { expr: Expression; sort: SortOrder; nulls?: NullsOrder }[], + ): AliasableExpression; override buildRelationSelection( query: SelectQueryBuilder, @@ -172,7 +176,8 @@ export abstract class LateralJoinDialectBase extends B ); if (relationFieldDef.array) { - return this.buildArrayAgg(this.buildJsonObject(objArgs)).as('$data'); + const orderBy = this.buildRelationOrderByExpressions(relationModel, relationModelAlias, payload); + return this.buildArrayAgg(this.buildJsonObject(objArgs), orderBy).as('$data'); } else { return this.buildJsonObject(objArgs).as('$data'); } @@ -181,6 +186,36 @@ export abstract class LateralJoinDialectBase extends B return qb; } + private buildRelationOrderByExpressions( + model: string, + modelAlias: string, + payload: true | FindArgs, any, true>, + ): { expr: Expression; sort: SortOrder; nulls?: NullsOrder }[] | undefined { + if (payload === true || !payload.orderBy) { + return undefined; + } + + type ScalarSortValue = SortOrder | { sort: SortOrder; nulls?: NullsOrder }; + const items: { expr: Expression; sort: SortOrder; nulls?: NullsOrder }[] = []; + + for (const orderBy of ensureArray(payload.orderBy)) { + for (const [field, value] of Object.entries(orderBy) as [string, ScalarSortValue | undefined][]) { + if (!value || requireField(this.schema, model, field).relation) { + continue; + } + + const expr = this.fieldRef(model, field, modelAlias); + if (typeof value === 'string') { + items.push({ expr, sort: value }); + } else { + items.push({ expr, sort: value.sort, nulls: value.nulls }); + } + } + } + + return items.length > 0 ? items : undefined; + } + private buildRelationObjectArgs( relationModel: string, relationModelAlias: string, diff --git a/packages/orm/src/client/crud/dialects/postgresql.ts b/packages/orm/src/client/crud/dialects/postgresql.ts index a601f8028..48d78b1d4 100644 --- a/packages/orm/src/client/crud/dialects/postgresql.ts +++ b/packages/orm/src/client/crud/dialects/postgresql.ts @@ -11,7 +11,7 @@ import { import { parse as parsePostgresArray } from 'postgres-array'; import { AnyNullClass, DbNullClass, JsonNullClass } from '../../../common-types'; import type { BuiltinType, FieldDef, SchemaDef } from '../../../schema'; -import type { SortOrder } from '../../crud-types'; +import type { NullsOrder, SortOrder } from '../../crud-types'; import { createInvalidInputError } from '../../errors'; import type { ClientOptions } from '../../options'; import { isEnum, isTypeDef } from '../../query-utils'; @@ -272,8 +272,24 @@ export class PostgresCrudDialect extends LateralJoinDi // #region other overrides - protected buildArrayAgg(arg: Expression) { - return this.eb.fn.coalesce(sql`jsonb_agg(${arg})`, sql`'[]'::jsonb`); + protected buildArrayAgg( + arg: Expression, + orderBy?: { expr: Expression; sort: SortOrder; nulls?: NullsOrder }[], + ) { + if (!orderBy || orderBy.length === 0) { + return this.eb.fn.coalesce(sql`jsonb_agg(${arg})`, sql`'[]'::jsonb`); + } + + const orderBySql = sql.join( + orderBy.map(({ expr, sort, nulls }) => { + const dir = sql.raw(sort.toUpperCase()); + const nullsSql = nulls ? sql` NULLS ${sql.raw(nulls.toUpperCase())}` : sql``; + return sql`${expr} ${dir}${nullsSql}`; + }), + sql.raw(', '), + ); + + return this.eb.fn.coalesce(sql`jsonb_agg(${arg} ORDER BY ${orderBySql})`, sql`'[]'::jsonb`); } override buildSkipTake( diff --git a/tests/e2e/orm/client-api/relation/order-by-nested-includes.test.ts b/tests/e2e/orm/client-api/relation/order-by-nested-includes.test.ts new file mode 100644 index 000000000..8639a7e7e --- /dev/null +++ b/tests/e2e/orm/client-api/relation/order-by-nested-includes.test.ts @@ -0,0 +1,156 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { createTestClient } from '@zenstackhq/testtools'; + +const TEST_DB = 'client-api-relation-test-order-by-nested-includes'; + +const schema = ` +model User { + id String @id + email String @unique + posts Post[] + comments Comment[] +} + +model Post { + id String @id + sequence Int + title String + author User @relation(fields: [authorId], references: [id]) + authorId String + comments Comment[] +} + +model Comment { + id String @id + content String + post Post @relation(fields: [postId], references: [id]) + postId String + author User? @relation(fields: [authorId], references: [id]) + authorId String? +} +`; + +function makePostsData(count: number) { + return Array.from({ length: count }, (_, i) => { + const sequence = count - i; // insert descending + return { + id: `p${sequence}`, + sequence, + title: `P${sequence}`, + // Keep outer relation (User -> posts) required. + authorId: 'u1', + }; + }); +} + +function makeCommentsData(count: number) { + return Array.from({ length: count }, (_, i) => { + const sequence = count - i; + return { + id: `c${sequence}`, + postId: `p${sequence}`, + content: `C${sequence}`, + // Make nested to-one include nullable to vary lateral join execution. + authorId: sequence % 11 === 0 ? null : 'u1', + }; + }); +} + +describe.each([{ provider: 'sqlite' as const }, { provider: 'postgresql' as const }])( + 'Relation orderBy with nested includes ($provider)', + ({ provider }) => { + let db: any; + + afterEach(async () => { + await db?.$disconnect(); + }); + + it('keeps stable order for to-many include with nested includes', async () => { + const count = provider === 'postgresql' ? 2000 : 10; + + db = await createTestClient(schema, { + provider, + dbName: `${TEST_DB}-${provider}-count-${count}`, + }); + + await db.user.create({ data: { id: 'u1', email: 'u1@example.com' } }); + await db.post.createMany({ data: makePostsData(count) }); + await db.comment.createMany({ data: makeCommentsData(count) }); + + const user = await db.user.findFirst({ + where: { id: 'u1' }, + include: { + posts: { + orderBy: { sequence: 'asc' }, + include: { author: true, comments: { include: { author: true } } }, + }, + }, + }); + + const ascSequences = user.posts.map((p: any) => p.sequence); + expect(ascSequences).toEqual(Array.from({ length: count }, (_, i) => i + 1)); + + const userDesc = await db.user.findFirst({ + where: { id: 'u1' }, + include: { + posts: { + orderBy: { sequence: 'desc' }, + include: { author: true, comments: { include: { author: true } } }, + }, + }, + }); + + const descSequences = userDesc.posts.map((p: any) => p.sequence) + expect(descSequences).toEqual(Array.from({ length: count }, (_, i) => count - i)); + }); + + it('keeps stable order for to-many select with nested selects', async () => { + const count = provider === 'postgresql' ? 2000 : 10; + + db = await createTestClient(schema, { + provider, + dbName: `${TEST_DB}-${provider}-select-count-${count}`, + }); + + await db.user.create({ data: { id: 'u1', email: 'u1@example.com' } }); + await db.post.createMany({ data: makePostsData(count) }); + await db.comment.createMany({ data: makeCommentsData(count) }); + + const user = await db.user.findFirst({ + where: { id: 'u1' }, + select: { + id: true, + posts: { + orderBy: { sequence: 'asc' }, + select: { + sequence: true, + author: { select: { id: true } }, + comments: { select: { author: { select: { id: true } } } }, + }, + }, + }, + }); + + const ascSequences = user.posts.map((p: any) => p.sequence); + expect(ascSequences).toEqual(Array.from({ length: count }, (_, i) => i + 1)); + + const userDesc = await db.user.findFirst({ + where: { id: 'u1' }, + select: { + id: true, + posts: { + orderBy: { sequence: 'desc' }, + select: { + sequence: true, + author: { select: { id: true } }, + comments: { select: { author: { select: { id: true } } } }, + }, + }, + }, + }); + + const descSequences = userDesc.posts.map((p: any) => p.sequence); + expect(descSequences).toEqual(Array.from({ length: count }, (_, i) => count - i)); + }); + }, +); From 1507f1895190f24ea54b34c841064fca74dd8865 Mon Sep 17 00:00:00 2001 From: Emil Redzik Date: Wed, 25 Mar 2026 14:10:01 +0100 Subject: [PATCH 2/4] fix: fixed ai review comments --- .../dialects/lateral-join-dialect-base.ts | 8 +++++ .../orm/src/client/crud/dialects/mysql.ts | 31 +++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/packages/orm/src/client/crud/dialects/lateral-join-dialect-base.ts b/packages/orm/src/client/crud/dialects/lateral-join-dialect-base.ts index ed2397795..5f4a4518e 100644 --- a/packages/orm/src/client/crud/dialects/lateral-join-dialect-base.ts +++ b/packages/orm/src/client/crud/dialects/lateral-join-dialect-base.ts @@ -186,6 +186,14 @@ export abstract class LateralJoinDialectBase extends B return qb; } + /** + * Extracts scalar `orderBy` clauses from the relation payload and maps them to + * the array-aggregation ordering format. + * + * For to-many relations aggregated into a JSON array (via lateral joins), this + * lets us preserve a stable ordering by passing `{ expr, sort, nulls? }` into + * the dialect's `buildArrayAgg` implementation. + */ private buildRelationOrderByExpressions( model: string, modelAlias: string, diff --git a/packages/orm/src/client/crud/dialects/mysql.ts b/packages/orm/src/client/crud/dialects/mysql.ts index 5533a59aa..dec2cf6b5 100644 --- a/packages/orm/src/client/crud/dialects/mysql.ts +++ b/packages/orm/src/client/crud/dialects/mysql.ts @@ -12,7 +12,7 @@ import { } from 'kysely'; import { AnyNullClass, DbNullClass, JsonNullClass } from '../../../common-types'; import type { BuiltinType, FieldDef, SchemaDef } from '../../../schema'; -import type { SortOrder } from '../../crud-types'; +import type { NullsOrder, SortOrder } from '../../crud-types'; import { createInvalidInputError, createNotSupportedError } from '../../errors'; import type { ClientOptions } from '../../options'; import { isTypeDef } from '../../query-utils'; @@ -192,8 +192,33 @@ export class MySqlCrudDialect extends LateralJoinDiale return this.eb.exists(this.eb.selectFrom(innerQuery.as('$exists_sub')).select(this.eb.lit(1).as('_'))); } - protected buildArrayAgg(arg: Expression): AliasableExpression { - return this.eb.fn.coalesce(sql`JSON_ARRAYAGG(${arg})`, sql`JSON_ARRAY()`); + protected buildArrayAgg( + arg: Expression, + orderBy?: { expr: Expression; sort: SortOrder; nulls?: NullsOrder }[], + ): AliasableExpression { + if (!orderBy || orderBy.length === 0) { + return this.eb.fn.coalesce(sql`JSON_ARRAYAGG(${arg})`, sql`JSON_ARRAY()`); + } + + // MySQL doesn't support explicit `NULLS FIRST|LAST` in ORDER BY, so emulate it + // by ordering on a boolean null-check first. + const orderBySql = sql.join( + orderBy.flatMap(({ expr, sort, nulls }) => { + const dir = sql.raw(sort.toUpperCase()); + if (!nulls) { + return [sql`${expr} ${dir}`]; + } + + const nullsSql = nulls === 'first' ? sql`(${expr} IS NULL) DESC` : sql`(${expr} IS NULL) ASC`; + return [nullsSql, sql`${expr} ${dir}`]; + }), + sql.raw(', '), + ); + + return this.eb.fn.coalesce( + sql`JSON_ARRAYAGG(${arg} ORDER BY ${orderBySql})`, + sql`JSON_ARRAY()`, + ); } override buildSkipTake( From c339fe769dd163c54eba36df9ae7af63a0e8c469 Mon Sep 17 00:00:00 2001 From: Emil Redzik Date: Fri, 27 Mar 2026 09:39:52 +0100 Subject: [PATCH 3/4] fix: fixed mysql query --- .../orm/src/client/crud/dialects/mysql.ts | 29 ++++--------------- .../relation/order-by-nested-includes.test.ts | 5 ++-- 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/packages/orm/src/client/crud/dialects/mysql.ts b/packages/orm/src/client/crud/dialects/mysql.ts index dec2cf6b5..dff577204 100644 --- a/packages/orm/src/client/crud/dialects/mysql.ts +++ b/packages/orm/src/client/crud/dialects/mysql.ts @@ -194,31 +194,12 @@ export class MySqlCrudDialect extends LateralJoinDiale protected buildArrayAgg( arg: Expression, - orderBy?: { expr: Expression; sort: SortOrder; nulls?: NullsOrder }[], + _orderBy?: { expr: Expression; sort: SortOrder; nulls?: NullsOrder }[], ): AliasableExpression { - if (!orderBy || orderBy.length === 0) { - return this.eb.fn.coalesce(sql`JSON_ARRAYAGG(${arg})`, sql`JSON_ARRAY()`); - } - - // MySQL doesn't support explicit `NULLS FIRST|LAST` in ORDER BY, so emulate it - // by ordering on a boolean null-check first. - const orderBySql = sql.join( - orderBy.flatMap(({ expr, sort, nulls }) => { - const dir = sql.raw(sort.toUpperCase()); - if (!nulls) { - return [sql`${expr} ${dir}`]; - } - - const nullsSql = nulls === 'first' ? sql`(${expr} IS NULL) DESC` : sql`(${expr} IS NULL) ASC`; - return [nullsSql, sql`${expr} ${dir}`]; - }), - sql.raw(', '), - ); - - return this.eb.fn.coalesce( - sql`JSON_ARRAYAGG(${arg} ORDER BY ${orderBySql})`, - sql`JSON_ARRAY()`, - ); + // MySQL doesn't support ORDER BY inside JSON_ARRAYAGG. + // For relation queries that need deterministic ordering, ordering is applied + // by the input subquery before aggregation. + return this.eb.fn.coalesce(sql`JSON_ARRAYAGG(${arg})`, sql`JSON_ARRAY()`); } override buildSkipTake( diff --git a/tests/e2e/orm/client-api/relation/order-by-nested-includes.test.ts b/tests/e2e/orm/client-api/relation/order-by-nested-includes.test.ts index 8639a7e7e..d7af30ec8 100644 --- a/tests/e2e/orm/client-api/relation/order-by-nested-includes.test.ts +++ b/tests/e2e/orm/client-api/relation/order-by-nested-includes.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, it } from 'vitest'; import { createTestClient } from '@zenstackhq/testtools'; -const TEST_DB = 'client-api-relation-test-order-by-nested-includes'; +const TEST_DB = 'order-by-nested-includes'; const schema = ` model User { @@ -56,7 +56,7 @@ function makeCommentsData(count: number) { }); } -describe.each([{ provider: 'sqlite' as const }, { provider: 'postgresql' as const }])( +describe.each([{ provider: 'sqlite' as const }, { provider: 'postgresql' as const }, {provider: 'mysql' as const}])( 'Relation orderBy with nested includes ($provider)', ({ provider }) => { let db: any; @@ -71,6 +71,7 @@ describe.each([{ provider: 'sqlite' as const }, { provider: 'postgresql' as cons db = await createTestClient(schema, { provider, dbName: `${TEST_DB}-${provider}-count-${count}`, + debug:true, }); await db.user.create({ data: { id: 'u1', email: 'u1@example.com' } }); From 16c196a67a8871736a5e33c1aa9b99d4e1e8bda0 Mon Sep 17 00:00:00 2001 From: ymc9 <104139426+ymc9@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:40:55 -0700 Subject: [PATCH 4/4] chore: clean up test file --- .../relation/order-by-nested-includes.test.ts | 166 ++++++++---------- 1 file changed, 77 insertions(+), 89 deletions(-) diff --git a/tests/e2e/orm/client-api/relation/order-by-nested-includes.test.ts b/tests/e2e/orm/client-api/relation/order-by-nested-includes.test.ts index d7af30ec8..a95fa6c6c 100644 --- a/tests/e2e/orm/client-api/relation/order-by-nested-includes.test.ts +++ b/tests/e2e/orm/client-api/relation/order-by-nested-includes.test.ts @@ -1,7 +1,5 @@ -import { afterEach, describe, expect, it } from 'vitest'; import { createTestClient } from '@zenstackhq/testtools'; - -const TEST_DB = 'order-by-nested-includes'; +import { afterEach, describe, expect, it } from 'vitest'; const schema = ` model User { @@ -56,102 +54,92 @@ function makeCommentsData(count: number) { }); } -describe.each([{ provider: 'sqlite' as const }, { provider: 'postgresql' as const }, {provider: 'mysql' as const}])( - 'Relation orderBy with nested includes ($provider)', - ({ provider }) => { - let db: any; +describe('Relation orderBy with nested includes', () => { + let db: any; - afterEach(async () => { - await db?.$disconnect(); - }); + afterEach(async () => { + await db?.$disconnect(); + }); - it('keeps stable order for to-many include with nested includes', async () => { - const count = provider === 'postgresql' ? 2000 : 10; - - db = await createTestClient(schema, { - provider, - dbName: `${TEST_DB}-${provider}-count-${count}`, - debug:true, - }); - - await db.user.create({ data: { id: 'u1', email: 'u1@example.com' } }); - await db.post.createMany({ data: makePostsData(count) }); - await db.comment.createMany({ data: makeCommentsData(count) }); - - const user = await db.user.findFirst({ - where: { id: 'u1' }, - include: { - posts: { - orderBy: { sequence: 'asc' }, - include: { author: true, comments: { include: { author: true } } }, - }, - }, - }); + it('keeps stable order for to-many include with nested includes', async () => { + const count = 2000; - const ascSequences = user.posts.map((p: any) => p.sequence); - expect(ascSequences).toEqual(Array.from({ length: count }, (_, i) => i + 1)); + db = await createTestClient(schema); - const userDesc = await db.user.findFirst({ - where: { id: 'u1' }, - include: { - posts: { - orderBy: { sequence: 'desc' }, - include: { author: true, comments: { include: { author: true } } }, - }, + await db.user.create({ data: { id: 'u1', email: 'u1@example.com' } }); + await db.post.createMany({ data: makePostsData(count) }); + await db.comment.createMany({ data: makeCommentsData(count) }); + + const user = await db.user.findFirst({ + where: { id: 'u1' }, + include: { + posts: { + orderBy: { sequence: 'asc' }, + include: { author: true, comments: { include: { author: true } } }, }, - }); + }, + }); - const descSequences = userDesc.posts.map((p: any) => p.sequence) - expect(descSequences).toEqual(Array.from({ length: count }, (_, i) => count - i)); + const ascSequences = user.posts.map((p: any) => p.sequence); + expect(ascSequences).toEqual(Array.from({ length: count }, (_, i) => i + 1)); + + const userDesc = await db.user.findFirst({ + where: { id: 'u1' }, + include: { + posts: { + orderBy: { sequence: 'desc' }, + include: { author: true, comments: { include: { author: true } } }, + }, + }, }); - it('keeps stable order for to-many select with nested selects', async () => { - const count = provider === 'postgresql' ? 2000 : 10; - - db = await createTestClient(schema, { - provider, - dbName: `${TEST_DB}-${provider}-select-count-${count}`, - }); - - await db.user.create({ data: { id: 'u1', email: 'u1@example.com' } }); - await db.post.createMany({ data: makePostsData(count) }); - await db.comment.createMany({ data: makeCommentsData(count) }); - - const user = await db.user.findFirst({ - where: { id: 'u1' }, - select: { - id: true, - posts: { - orderBy: { sequence: 'asc' }, - select: { - sequence: true, - author: { select: { id: true } }, - comments: { select: { author: { select: { id: true } } } }, - }, + const descSequences = userDesc.posts.map((p: any) => p.sequence); + expect(descSequences).toEqual(Array.from({ length: count }, (_, i) => count - i)); + }); + + it('keeps stable order for to-many select with nested selects', async () => { + const count = 2000; + + db = await createTestClient(schema); + + await db.user.create({ data: { id: 'u1', email: 'u1@example.com' } }); + await db.post.createMany({ data: makePostsData(count) }); + await db.comment.createMany({ data: makeCommentsData(count) }); + + const user = await db.user.findFirst({ + where: { id: 'u1' }, + select: { + id: true, + posts: { + orderBy: { sequence: 'asc' }, + select: { + sequence: true, + author: { select: { id: true } }, + comments: { select: { author: { select: { id: true } } } }, }, }, - }); - - const ascSequences = user.posts.map((p: any) => p.sequence); - expect(ascSequences).toEqual(Array.from({ length: count }, (_, i) => i + 1)); - - const userDesc = await db.user.findFirst({ - where: { id: 'u1' }, - select: { - id: true, - posts: { - orderBy: { sequence: 'desc' }, - select: { - sequence: true, - author: { select: { id: true } }, - comments: { select: { author: { select: { id: true } } } }, - }, + }, + }); + + const ascSequences = user.posts.map((p: any) => p.sequence); + expect(ascSequences).toEqual(Array.from({ length: count }, (_, i) => i + 1)); + + const userDesc = await db.user.findFirst({ + where: { id: 'u1' }, + select: { + id: true, + posts: { + orderBy: { sequence: 'desc' }, + select: { + sequence: true, + author: { select: { id: true } }, + comments: { select: { author: { select: { id: true } } } }, }, }, - }); - - const descSequences = userDesc.posts.map((p: any) => p.sequence); - expect(descSequences).toEqual(Array.from({ length: count }, (_, i) => count - i)); + }, }); - }, -); + + const descSequences = userDesc.posts.map((p: any) => p.sequence); + expect(descSequences).toEqual(Array.from({ length: count }, (_, i) => count - i)); + }); +});