Skip to content

Commit f93c7c1

Browse files
JanTvrdikJanTvrdik
authored
EntityPreloader: add support for preloading many to many associations (#4)
* EntityPreloader: add support for preloading many to many associations * EntityPreloader: refactor preloadOneToMany * EntityPreloader: refactor preloading has many associations * add EntityPreloadBlogManyHasManyInversedTest
1 parent e78a8c6 commit f93c7c1

File tree

5 files changed

+362
-38
lines changed

5 files changed

+362
-38
lines changed

‎composer.json‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@
66
],
77
"require": {
88
"php": "^8.1",
9-
"doctrine/orm": "^3",
10-
"doctrine/persistence": "^3.1"
9+
"doctrine/orm": "^3"
1110
},
1211
"require-dev": {
1312
"doctrine/collections": "^2.2",

‎src/EntityPreloader.php‎

Lines changed: 155 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,9 @@
66
use Doctrine\ORM\Mapping\ClassMetadata;
77
use Doctrine\ORM\PersistentCollection;
88
use Doctrine\ORM\QueryBuilder;
9-
use Doctrine\Persistence\Proxy;
109
use LogicException;
10+
use ReflectionProperty;
1111
use function array_chunk;
12-
use function array_keys;
1312
use function array_values;
1413
use function count;
1514
use function get_parent_class;
@@ -21,7 +20,7 @@
2120
class EntityPreloader
2221
{
2322

24-
private const BATCH_SIZE = 1_000;
23+
private const PRELOAD_ENTITY_DEFAULT_BATCH_SIZE = 1_000;
2524
private const PRELOAD_COLLECTION_DEFAULT_BATCH_SIZE = 100;
2625

2726
public function __construct(
@@ -65,14 +64,15 @@ public function preload(
6564
}
6665

6766
$maxFetchJoinSameFieldCount ??= 1;
68-
$sourceEntities = $this->loadProxies($sourceClassMetadata, $sourceEntities, $batchSize ?? self::BATCH_SIZE, $maxFetchJoinSameFieldCount);
67+
$sourceEntities = $this->loadProxies($sourceClassMetadata, $sourceEntities, $batchSize ?? self::PRELOAD_ENTITY_DEFAULT_BATCH_SIZE, $maxFetchJoinSameFieldCount);
6968

70-
return match ($associationMapping->type()) {
71-
ClassMetadata::ONE_TO_MANY => $this->preloadOneToMany($sourceEntities, $sourceClassMetadata, $sourcePropertyName, $targetClassMetadata, $batchSize, $maxFetchJoinSameFieldCount),
72-
ClassMetadata::ONE_TO_ONE,
73-
ClassMetadata::MANY_TO_ONE => $this->preloadToOne($sourceEntities, $sourceClassMetadata, $sourcePropertyName, $targetClassMetadata, $batchSize, $maxFetchJoinSameFieldCount),
69+
$preloader = match (true) {
70+
$associationMapping->isToOne() => $this->preloadToOne(...),
71+
$associationMapping->isToMany() => $this->preloadToMany(...),
7472
default => throw new LogicException("Unsupported association mapping type {$associationMapping->type()}"),
7573
};
74+
75+
return $preloader($sourceEntities, $sourceClassMetadata, $sourcePropertyName, $targetClassMetadata, $batchSize, $maxFetchJoinSameFieldCount);
7676
}
7777

7878
/**
@@ -135,7 +135,7 @@ private function loadProxies(
135135
$entityKey = (string) $entityId;
136136
$uniqueEntities[$entityKey] = $entity;
137137

138-
if ($entity instanceof Proxy && !$entity->__isInitialized()) {
138+
if ($this->entityManager->isUninitializedObject($entity)) {
139139
$uninitializedIds[$entityKey] = $entityId;
140140
}
141141
}
@@ -157,7 +157,7 @@ private function loadProxies(
157157
* @template S of E
158158
* @template T of E
159159
*/
160-
private function preloadOneToMany(
160+
private function preloadToMany(
161161
array $sourceEntities,
162162
ClassMetadata $sourceClassMetadata,
163163
string $sourcePropertyName,
@@ -168,50 +168,170 @@ private function preloadOneToMany(
168168
{
169169
$sourceIdentifierReflection = $sourceClassMetadata->getSingleIdReflectionProperty(); // e.g. Order::$id reflection
170170
$sourcePropertyReflection = $sourceClassMetadata->getReflectionProperty($sourcePropertyName); // e.g. Order::$items reflection
171-
$targetPropertyName = $sourceClassMetadata->getAssociationMappedByTargetField($sourcePropertyName); // e.g. 'order'
172-
$targetPropertyReflection = $targetClassMetadata->getReflectionProperty($targetPropertyName); // e.g. Item::$order reflection
171+
$targetIdentifierReflection = $targetClassMetadata->getSingleIdReflectionProperty();
173172

174-
if ($sourceIdentifierReflection === null || $sourcePropertyReflection === null || $targetPropertyReflection === null) {
173+
if ($sourceIdentifierReflection === null || $sourcePropertyReflection === null || $targetIdentifierReflection === null) {
175174
throw new LogicException('Doctrine should use RuntimeReflectionService which never returns null.');
176175
}
177176

178177
$batchSize ??= self::PRELOAD_COLLECTION_DEFAULT_BATCH_SIZE;
179-
180178
$targetEntities = [];
179+
$uninitializedSourceEntityIds = [];
181180
$uninitializedCollections = [];
182181

183182
foreach ($sourceEntities as $sourceEntity) {
184-
$sourceEntityId = (string) $sourceIdentifierReflection->getValue($sourceEntity);
183+
$sourceEntityId = $sourceIdentifierReflection->getValue($sourceEntity);
184+
$sourceEntityKey = (string) $sourceEntityId;
185185
$sourceEntityCollection = $sourcePropertyReflection->getValue($sourceEntity);
186186

187187
if (
188188
$sourceEntityCollection instanceof PersistentCollection
189189
&& !$sourceEntityCollection->isInitialized()
190190
&& !$sourceEntityCollection->isDirty() // preloading dirty collection is too hard to handle
191191
) {
192-
$uninitializedCollections[$sourceEntityId] = $sourceEntityCollection;
192+
$uninitializedSourceEntityIds[$sourceEntityKey] = $sourceEntityId;
193+
$uninitializedCollections[$sourceEntityKey] = $sourceEntityCollection;
193194
continue;
194195
}
195196

196197
foreach ($sourceEntityCollection as $targetEntity) {
197-
$targetEntities[] = $targetEntity;
198+
$targetEntityKey = (string) $targetIdentifierReflection->getValue($targetEntity);
199+
$targetEntities[$targetEntityKey] = $targetEntity;
198200
}
199201
}
200202

201-
foreach (array_chunk($uninitializedCollections, $batchSize, true) as $chunk) {
202-
$targetEntitiesChunk = $this->loadEntitiesBy($targetClassMetadata, $targetPropertyName, array_keys($chunk), $maxFetchJoinSameFieldCount);
203+
$innerLoader = match ($sourceClassMetadata->getAssociationMapping($sourcePropertyName)->type()) {
204+
ClassMetadata::ONE_TO_MANY => $this->preloadOneToManyInner(...),
205+
ClassMetadata::MANY_TO_MANY => $this->preloadManyToManyInner(...),
206+
default => throw new LogicException('Unsupported association mapping type'),
207+
};
203208

204-
foreach ($targetEntitiesChunk as $targetEntity) {
205-
$sourceEntity = $targetPropertyReflection->getValue($targetEntity);
206-
$sourceEntityId = (string) $sourceIdentifierReflection->getValue($sourceEntity);
207-
$uninitializedCollections[$sourceEntityId]->add($targetEntity);
208-
$targetEntities[] = $targetEntity;
209+
foreach (array_chunk($uninitializedSourceEntityIds, $batchSize, preserve_keys: true) as $uninitializedSourceEntityIdsChunk) {
210+
$targetEntitiesChunk = $innerLoader(
211+
sourceClassMetadata: $sourceClassMetadata,
212+
sourceIdentifierReflection: $sourceIdentifierReflection,
213+
sourcePropertyName: $sourcePropertyName,
214+
targetClassMetadata: $targetClassMetadata,
215+
targetIdentifierReflection: $targetIdentifierReflection,
216+
uninitializedSourceEntityIdsChunk: array_values($uninitializedSourceEntityIdsChunk),
217+
uninitializedCollections: $uninitializedCollections,
218+
maxFetchJoinSameFieldCount: $maxFetchJoinSameFieldCount,
219+
);
220+
221+
foreach ($targetEntitiesChunk as $targetEntityKey => $targetEntity) {
222+
$targetEntities[$targetEntityKey] = $targetEntity;
209223
}
224+
}
225+
226+
foreach ($uninitializedCollections as $sourceEntityCollection) {
227+
$sourceEntityCollection->setInitialized(true);
228+
$sourceEntityCollection->takeSnapshot();
229+
}
230+
231+
return array_values($targetEntities);
232+
}
233+
234+
/**
235+
* @param ClassMetadata<S> $sourceClassMetadata
236+
* @param ClassMetadata<T> $targetClassMetadata
237+
* @param list<mixed> $uninitializedSourceEntityIdsChunk
238+
* @param array<string, PersistentCollection<int, T>> $uninitializedCollections
239+
* @param non-negative-int $maxFetchJoinSameFieldCount
240+
* @return array<string, T>
241+
* @template S of E
242+
* @template T of E
243+
*/
244+
private function preloadOneToManyInner(
245+
ClassMetadata $sourceClassMetadata,
246+
ReflectionProperty $sourceIdentifierReflection,
247+
string $sourcePropertyName,
248+
ClassMetadata $targetClassMetadata,
249+
ReflectionProperty $targetIdentifierReflection,
250+
array $uninitializedSourceEntityIdsChunk,
251+
array $uninitializedCollections,
252+
int $maxFetchJoinSameFieldCount,
253+
): array
254+
{
255+
$targetPropertyName = $sourceClassMetadata->getAssociationMappedByTargetField($sourcePropertyName); // e.g. 'order'
256+
$targetPropertyReflection = $targetClassMetadata->getReflectionProperty($targetPropertyName); // e.g. Item::$order reflection
257+
$targetEntities = [];
258+
259+
if ($targetPropertyReflection === null) {
260+
throw new LogicException('Doctrine should use RuntimeReflectionService which never returns null.');
261+
}
262+
263+
foreach ($this->loadEntitiesBy($targetClassMetadata, $targetPropertyName, $uninitializedSourceEntityIdsChunk, $maxFetchJoinSameFieldCount) as $targetEntity) {
264+
$sourceEntity = $targetPropertyReflection->getValue($targetEntity);
265+
$sourceEntityKey = (string) $sourceIdentifierReflection->getValue($sourceEntity);
266+
$uninitializedCollections[$sourceEntityKey]->add($targetEntity);
267+
268+
$targetEntityKey = (string) $targetIdentifierReflection->getValue($targetEntity);
269+
$targetEntities[$targetEntityKey] = $targetEntity;
270+
}
210271

211-
foreach ($chunk as $sourceEntityCollection) {
212-
$sourceEntityCollection->setInitialized(true);
213-
$sourceEntityCollection->takeSnapshot();
272+
return $targetEntities;
273+
}
274+
275+
/**
276+
* @param ClassMetadata<S> $sourceClassMetadata
277+
* @param ClassMetadata<T> $targetClassMetadata
278+
* @param list<mixed> $uninitializedSourceEntityIdsChunk
279+
* @param array<string, PersistentCollection<int, T>> $uninitializedCollections
280+
* @param non-negative-int $maxFetchJoinSameFieldCount
281+
* @return array<string, T>
282+
* @template S of E
283+
* @template T of E
284+
*/
285+
private function preloadManyToManyInner(
286+
ClassMetadata $sourceClassMetadata,
287+
ReflectionProperty $sourceIdentifierReflection,
288+
string $sourcePropertyName,
289+
ClassMetadata $targetClassMetadata,
290+
ReflectionProperty $targetIdentifierReflection,
291+
array $uninitializedSourceEntityIdsChunk,
292+
array $uninitializedCollections,
293+
int $maxFetchJoinSameFieldCount,
294+
): array
295+
{
296+
$sourceIdentifierName = $sourceClassMetadata->getSingleIdentifierFieldName();
297+
$targetIdentifierName = $targetClassMetadata->getSingleIdentifierFieldName();
298+
299+
$manyToManyRows = $this->entityManager->createQueryBuilder()
300+
->select("source.{$sourceIdentifierName} AS sourceId", "target.{$targetIdentifierName} AS targetId")
301+
->from($sourceClassMetadata->getName(), 'source')
302+
->join("source.{$sourcePropertyName}", 'target')
303+
->andWhere('source IN (:sourceEntityIds)')
304+
->setParameter('sourceEntityIds', $uninitializedSourceEntityIdsChunk)
305+
->getQuery()
306+
->getResult();
307+
308+
$targetEntities = [];
309+
$uninitializedTargetEntityIds = [];
310+
311+
foreach ($manyToManyRows as $manyToManyRow) {
312+
$targetEntityId = $manyToManyRow['targetId'];
313+
$targetEntityKey = (string) $targetEntityId;
314+
315+
/** @var T|false $targetEntity */
316+
$targetEntity = $this->entityManager->getUnitOfWork()->tryGetById($targetEntityId, $targetClassMetadata->getName());
317+
318+
if ($targetEntity !== false && !$this->entityManager->isUninitializedObject($targetEntity)) {
319+
$targetEntities[$targetEntityKey] = $targetEntity;
320+
continue;
214321
}
322+
323+
$uninitializedTargetEntityIds[$targetEntityKey] = $targetEntityId;
324+
}
325+
326+
foreach ($this->loadEntitiesBy($targetClassMetadata, $targetIdentifierName, array_values($uninitializedTargetEntityIds), $maxFetchJoinSameFieldCount) as $targetEntity) {
327+
$targetEntityKey = (string) $targetIdentifierReflection->getValue($targetEntity);
328+
$targetEntities[$targetEntityKey] = $targetEntity;
329+
}
330+
331+
foreach ($manyToManyRows as $manyToManyRow) {
332+
$sourceEntityKey = (string) $manyToManyRow['sourceId'];
333+
$targetEntityKey = (string) $manyToManyRow['targetId'];
334+
$uninitializedCollections[$sourceEntityKey]->add($targetEntities[$targetEntityKey]);
215335
}
216336

217337
return $targetEntities;
@@ -237,12 +357,14 @@ private function preloadToOne(
237357
): array
238358
{
239359
$sourcePropertyReflection = $sourceClassMetadata->getReflectionProperty($sourcePropertyName); // e.g. Item::$order reflection
240-
$targetEntities = [];
241360

242361
if ($sourcePropertyReflection === null) {
243362
throw new LogicException('Doctrine should use RuntimeReflectionService which never returns null.');
244363
}
245364

365+
$batchSize ??= self::PRELOAD_ENTITY_DEFAULT_BATCH_SIZE;
366+
$targetEntities = [];
367+
246368
foreach ($sourceEntities as $sourceEntity) {
247369
$targetEntity = $sourcePropertyReflection->getValue($sourceEntity);
248370

@@ -253,7 +375,7 @@ private function preloadToOne(
253375
$targetEntities[] = $targetEntity;
254376
}
255377

256-
return $this->loadProxies($targetClassMetadata, $targetEntities, $batchSize ?? self::BATCH_SIZE, $maxFetchJoinSameFieldCount);
378+
return $this->loadProxies($targetClassMetadata, $targetEntities, $batchSize, $maxFetchJoinSameFieldCount);
257379
}
258380

259381
/**
@@ -270,6 +392,10 @@ private function loadEntitiesBy(
270392
int $maxFetchJoinSameFieldCount,
271393
): array
272394
{
395+
if (count($fieldValues) === 0) {
396+
return [];
397+
}
398+
273399
$rootLevelAlias = 'e';
274400

275401
$queryBuilder = $this->entityManager->createQueryBuilder()

‎tests/EntityPreloadBlogManyHasManyInversedTest.php‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
<?php declare(strict_types = 1);
2+
3+
namespace ShipMonkTests\DoctrineEntityPreloader;
4+
5+
use Doctrine\ORM\Mapping\ClassMetadata;
6+
use ShipMonkTests\DoctrineEntityPreloader\Fixtures\Blog\Tag;
7+
use ShipMonkTests\DoctrineEntityPreloader\Lib\TestCase;
8+
9+
class EntityPreloadBlogManyHasManyInversedTest extends TestCase
10+
{
11+
12+
public function testManyHasManyInversedUnoptimized(): void
13+
{
14+
$this->createDummyBlogData(articleInEachCategoryCount: 5, tagForEachArticleCount: 5);
15+
16+
$tags = $this->getEntityManager()->getRepository(Tag::class)->findAll();
17+
18+
$this->readArticleTitles($tags);
19+
20+
self::assertAggregatedQueries([
21+
['count' => 1, 'query' => 'SELECT * FROM tag t0'],
22+
['count' => 25, 'query' => 'SELECT * FROM article t0 INNER JOIN article_tag ON t0.id = article_tag.article_id WHERE article_tag.tag_id = ?'],
23+
]);
24+
}
25+
26+
public function testManyHasManyInversedWithFetchJoin(): void
27+
{
28+
$this->createDummyBlogData(articleInEachCategoryCount: 5, tagForEachArticleCount: 5);
29+
30+
$tags = $this->getEntityManager()->createQueryBuilder()
31+
->select('tag', 'article')
32+
->from(Tag::class, 'tag')
33+
->leftJoin('tag.articles', 'article')
34+
->getQuery()
35+
->getResult();
36+
37+
$this->readArticleTitles($tags);
38+
39+
self::assertAggregatedQueries([
40+
['count' => 1, 'query' => 'SELECT * FROM tag t0_ LEFT JOIN article_tag a2_ ON t0_.id = a2_.tag_id LEFT JOIN article a1_ ON a1_.id = a2_.article_id'],
41+
]);
42+
}
43+
44+
public function testManyHasManyInversedWithEagerFetchMode(): void
45+
{
46+
$this->createDummyBlogData(articleInEachCategoryCount: 5, tagForEachArticleCount: 5);
47+
48+
// for eagerly loaded Many-To-Many associations one query has to be made for each collection
49+
// https://www.doctrine-project.org/projects/doctrine-orm/en/3.2/reference/working-with-objects.html#by-eager-loading
50+
$tags = $this->getEntityManager()->createQueryBuilder()
51+
->select('tag')
52+
->from(Tag::class, 'tag')
53+
->getQuery()
54+
->setFetchMode(Tag::class, 'articles', ClassMetadata::FETCH_EAGER)
55+
->getResult();
56+
57+
$this->readArticleTitles($tags);
58+
59+
self::assertAggregatedQueries([
60+
['count' => 1, 'query' => 'SELECT * FROM tag t0_'],
61+
['count' => 25, 'query' => 'SELECT * FROM article t0 INNER JOIN article_tag ON t0.id = article_tag.article_id WHERE article_tag.tag_id = ?'],
62+
]);
63+
}
64+
65+
public function testManyHasManyInversedWithPreload(): void
66+
{
67+
$this->createDummyBlogData(articleInEachCategoryCount: 5, tagForEachArticleCount: 5);
68+
69+
$tags = $this->getEntityManager()->getRepository(Tag::class)->findAll();
70+
$this->getEntityPreloader()->preload($tags, 'articles');
71+
72+
$this->readArticleTitles($tags);
73+
74+
self::assertAggregatedQueries([
75+
['count' => 1, 'query' => 'SELECT * FROM tag t0'],
76+
['count' => 1, 'query' => 'SELECT * FROM tag t0_ INNER JOIN article_tag a2_ ON t0_.id = a2_.tag_id INNER JOIN article a1_ ON a1_.id = a2_.article_id WHERE t0_.id IN (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'],
77+
['count' => 1, 'query' => 'SELECT * FROM article a0_ WHERE a0_.id IN (?, ?, ?, ?, ?)'],
78+
]);
79+
}
80+
81+
/**
82+
* @param array<Tag> $tags
83+
*/
84+
private function readArticleTitles(array $tags): void
85+
{
86+
foreach ($tags as $tag) {
87+
foreach ($tag->getArticles() as $article) {
88+
$article->getTitle();
89+
}
90+
}
91+
}
92+
93+
}

0 commit comments

Comments
 (0)