Run the benchmark on PHP 7.4 in addition to PHP 8.5

.github/workflows/bench.yml

@@ -23,10 +23,17 @@
 
 jobs:
   baseline:
-    name: "Baseline"
+    name: "Baseline (PHP ${{ matrix.php-version }})"
     runs-on: ubuntu-latest
     timeout-minutes: 60
 
+    strategy:
+      fail-fast: false
+      matrix:
+        php-version:
+          - "7.4"
+          - "8.5"
+
     steps:
       - name: Harden the runner (Audit all outbound calls)
         uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
@@ -40,31 +47,65 @@
         uses: "shivammathur/setup-php@7c071dfe9dc99bdf297fa79cb49ea005b9fcadbc" # v2.37.1
         with:
           coverage: "none"
-          php-version: "8.5"
+          php-version: "${{ matrix.php-version }}"
           tools: pecl
           extensions: ds,mbstring
           ini-file: development
           ini-values: memory_limit=-1
 
+      - name: "Downgrade PHPUnit"
+        if: matrix.php-version == '7.4'
+        run: "composer require --dev phpunit/phpunit:^9.6 sebastian/diff:^4.0 doctrine/instantiator:^1.0 --update-with-dependencies --ignore-platform-reqs"
+
       - uses: "ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda" # v4.0.0
+
+      # tests/composer.json pins the Composer platform to PHP 8.2, which would make
+      # Composer resolve PHPBench's Symfony dependencies to versions that cannot
+      # even be parsed on PHP 7.4. Unsetting it resolves against the real PHP version.
+      - name: "Downgrade PHPBench"
+        if: matrix.php-version == '7.4'
+        run: |
+          composer config --unset platform.php --working-dir=tests
+          composer require --dev phpbench/phpbench:1.2.14 phpunit/phpunit:^9.6 brianium/paratest:^6.5 --update-with-all-dependencies --working-dir=tests
+
       - uses: "ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda" # v4.0.0
+        if: matrix.php-version != '7.4'
         with:
           working-directory: "tests/"
 
+      - uses: ./.github/actions/downgrade-code
+        with:
+          php-version: "${{ matrix.php-version }}"
+
       - name: "Run phpbench baseline"
-        run: "tests/vendor/bin/phpbench run --dump-file=tests/bench/storage/baseline.xml --ansi"
+        run: "tests/vendor/bin/phpbench run --dump-file=tests/bench/storage/baseline-${{ matrix.php-version }}.xml --ansi"
 
+      # Before committing an artifact as the new baseline, compare the <baseline-machine>
+      # values (md5, file_rw) in it with the ones in the currently committed baseline.
+      # GitHub runners differ in speed by up to 2x, and a baseline recorded on a fast
+      # runner makes every later run look like a regression.
       - name: "Upload baseline artifact"
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: phpbench-baseline
-          path: tests/bench/storage/baseline.xml
+          name: phpbench-baseline-${{ matrix.php-version }}
+          path: tests/bench/storage/baseline-${{ matrix.php-version }}.xml
 
   test:
-    name: "Test"
+    name: "Test (PHP ${{ matrix.php-version }})"
     runs-on: ubuntu-latest
     timeout-minutes: 60
 
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # phpbench 1.2.14 is the last version supporting PHP 7.4
+          # and it does not have the aggregate-preview report 'my-report' extends
+          - php-version: "7.4"
+            report: "aggregate"
+          - php-version: "8.5"
+            report: "my-report"
+
     steps:
       - name: Harden the runner (Audit all outbound calls)
         uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
@@ -78,16 +119,42 @@
         uses: "shivammathur/setup-php@7c071dfe9dc99bdf297fa79cb49ea005b9fcadbc" # v2.37.1
         with:
           coverage: "none"
-          php-version: "8.5"
+          php-version: "${{ matrix.php-version }}"
           tools: pecl
           extensions: ds,mbstring
           ini-file: development
           ini-values: memory_limit=-1
 
+      - name: "Downgrade PHPUnit"
+        if: matrix.php-version == '7.4'
+        run: "composer require --dev phpunit/phpunit:^9.6 sebastian/diff:^4.0 doctrine/instantiator:^1.0 --update-with-dependencies --ignore-platform-reqs"
+
       - uses: "ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda" # v4.0.0
+
+      # tests/composer.json pins the Composer platform to PHP 8.2, which would make
+      # Composer resolve PHPBench's Symfony dependencies to versions that cannot
+      # even be parsed on PHP 7.4. Unsetting it resolves against the real PHP version.
+      - name: "Downgrade PHPBench"
+        if: matrix.php-version == '7.4'
+        run: |
+          composer config --unset platform.php --working-dir=tests
+          composer require --dev phpbench/phpbench:1.2.14 phpunit/phpunit:^9.6 brianium/paratest:^6.5 --update-with-all-dependencies --working-dir=tests
+
       - uses: "ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda" # v4.0.0
+        if: matrix.php-version != '7.4'
         with:
           working-directory: "tests/"
 
+      - uses: ./.github/actions/downgrade-code
+        with:
+          php-version: "${{ matrix.php-version }}"
+
+      # PHPBench does not fail when the baseline file is missing, it just runs
+      # with an empty baseline and every assertion passes
+      - name: "Check that the baseline exists"
+        run: |
+          test -f tests/bench/storage/baseline-${{ matrix.php-version }}.xml \
+            || (echo "Missing tests/bench/storage/baseline-${{ matrix.php-version }}.xml, download it from the phpbench-baseline-${{ matrix.php-version }} artifact of the Baseline job and commit it." && exit 1)
+
       - name: "Run phpbench test"
-        run: "tests/vendor/bin/phpbench run --file=tests/bench/storage/baseline.xml --report=my-report --ansi"
+        run: "tests/vendor/bin/phpbench run --file=tests/bench/storage/baseline-${{ matrix.php-version }}.xml --report=${{ matrix.report }} --ansi"

Makefile

@@ -15,7 +15,6 @@ lint:
 	XDEBUG_MODE=off php vendor/bin/parallel-lint --colors \
 		--exclude tests/PHPStan/Analyser/data \
 		--exclude tests/bench/data \
-		--exclude tests/bench/RegressionBench.php \
 		--exclude tests/PHPStan/Analyser/nsrt \
 		--exclude tests/PHPStan/Rules/Methods/data \
 		--exclude tests/PHPStan/Rules/Functions/data \

src/DependencyInjection/Configurator.php

@@ -11,7 +11,6 @@
 use PHPStan\File\CouldNotWriteFileException;
 use PHPStan\File\FileReader;
 use PHPStan\File\FileWriter;
-use PHPStan\Turbo\TurboExtensionEnabler;
 use function array_keys;
 use function count;
 use function error_reporting;
@@ -30,7 +29,6 @@
 use function time;
 use function trim;
 use function unlink;
-use function var_export;
 use const E_USER_DEPRECATED;
 use const PHP_RELEASE_VERSION;
 use const PHP_VERSION_ID;
@@ -105,12 +103,6 @@ public function loadContainer(): string
 			is_file($attributesPhp) ? hash_file('sha256', $attributesPhp) : 'attributes-missing',
 			NeonAdapter::CACHE_KEY,
 			$this->getAllConfigFilesHashes(),
-			// the stubs, not the extension: a shadowed service class is
-			// reflected while the container compiles, so a container built
-			// against the stub shells must not be reused by a run that loaded
-			// the extension but left it inactive (version mismatch,
-			// PHPSTAN_TURBO=0) and reflects the PHP implementations instead
-			var_export(TurboExtensionEnabler::isActive(), true),
 		];
 
 		$className = $loader->load(

tests/bench/BenchCase.php

@@ -2,10 +2,20 @@
 
 namespace PHPStan\Benchmark;
 
+use LogicException;
 use PHPStan\Analyser\Analyser;
 use PHPStan\Analyser\AnalyserResultFinalizer;
 use PHPStan\Analyser\Error;
 use PHPStan\Testing\PHPStanTestCaseTrait;
+use function fclose;
+use function fgets;
+use function fopen;
+use function preg_match;
+use function sprintf;
+use function str_contains;
+use function str_starts_with;
+use function version_compare;
+use const PHP_VERSION;
 
 abstract class BenchCase
 {
@@ -19,6 +29,44 @@ public static function getAdditionalConfigFiles(): array
 		];
 	}
 
+	/**
+	 * Data files that cannot be parsed on the current PHP version are not worth
+	 * benchmarking - the analysis stops at the syntax error.
+	 *
+	 * Copy of TypeInferenceTestCase::isFileLintSkipped(), originally from
+	 * https://github.com/php-parallel-lint/PHP-Parallel-Lint/blob/0c2706086ac36dce31967cb36062ff8915fe03f7/bin/skip-linting.php
+	 *
+	 * Copyright (c) 2012, Jakub Onderka
+	 */
+	protected static function isFileLintSkipped(string $file): bool
+	{
+		$f = @fopen($file, 'r');
+		if ($f !== false) {
+			$firstLine = fgets($f);
+			if ($firstLine === false) {
+				return false;
+			}
+
+			// ignore shebang line
+			if (str_starts_with($firstLine, '#!')) {
+				$firstLine = fgets($f);
+				if ($firstLine === false) {
+					return false;
+				}
+			}
+
+			@fclose($f);
+
+			if (preg_match('~<?php\\s*\\/\\/\s*lint\s*([^\d\s]+)\s*([^\s]+)\s*~i', $firstLine, $m) === 1) {
+				return version_compare(PHP_VERSION, $m[2], $m[1]) === false;
+			} elseif (str_contains($firstLine, 'lint')) {
+				throw new LogicException(sprintf("'// lint' comment must immediately follow the php starting tag in %s on line 1", $file));
+			}
+		}
+
+		return false;
+	}
+
 	/**
 	 * @param string[]|null $allAnalysedFiles
 	 * @return list<Error>

tests/bench/RegressionBench.php

@@ -2,26 +2,31 @@
 
 namespace PHPStan\Benchmark;
 
-use PhpBench\Attributes as Bench;
 use Symfony\Component\Finder\Finder;
 
-#[Bench\Revs(revs: 1)]
-#[Bench\Iterations(iterations: 5)]
-#[Bench\Warmup(revs: 1)]
-#[Bench\RetryThreshold(retryThreshold: 10.0)]
-#[Bench\Assert(expression: '
-    (mode(baseline.time.avg) < 100 milliseconds and mode(variant.time.avg) < mode(baseline.time.avg) +/- 50%)
-    or (mode(baseline.time.avg) >= 100 milliseconds and mode(baseline.time.avg) < 500 milliseconds and mode(variant.time.avg) < mode(baseline.time.avg) +/- 25%)
-    or (mode(baseline.time.avg) >= 500 milliseconds and mode(baseline.time.avg) < 2000 milliseconds and mode(variant.time.avg) < mode(baseline.time.avg) +/- 20%)
-    or (mode(baseline.time.avg) >= 2000 milliseconds and mode(variant.time.avg) < mode(baseline.time.avg) +/- 10%)
-')]
+/**
+ * PHPBench annotations are used instead of attributes so that the benchmark
+ * also runs on PHP 7.4 with the downgraded source code, where attributes
+ * are just comments.
+ *
+ * The assertion expression has to stay on a single line - an annotation value
+ * cannot span multiple lines. Annotation names must not be mentioned anywhere
+ * else in this docblock either, the annotation reader tries to parse them.
+ *
+ * @Revs(1)
+ * @Iterations(5)
+ * @Warmup(1)
+ * @RetryThreshold(10.0)
+ * @Assert("(mode(baseline.time.avg) < 100 milliseconds and mode(variant.time.avg) < mode(baseline.time.avg) +/- 50%) or (mode(baseline.time.avg) >= 100 milliseconds and mode(baseline.time.avg) < 500 milliseconds and mode(variant.time.avg) < mode(baseline.time.avg) +/- 25%) or (mode(baseline.time.avg) >= 500 milliseconds and mode(baseline.time.avg) < 2000 milliseconds and mode(variant.time.avg) < mode(baseline.time.avg) +/- 20%) or (mode(baseline.time.avg) >= 2000 milliseconds and mode(variant.time.avg) < mode(baseline.time.avg) +/- 10%)")
+ */
 class RegressionBench extends BenchCase
 {
 
 	/**
+	 * @ParamProviders({"provideFiles"})
+	 *
 	 * @param array{string} $params
 	 */
-	#[Bench\ParamProviders(['provideFiles'])]
 	public function benchRunAnalyse(array $params): void
 	{
 		$this->runAnalyse($params[0]);
@@ -42,6 +47,10 @@ private static function findTestDataFilesFromDirectory(string $directory): array
 		$finder->sortByName(true);
 		$files = [];
 		foreach ($finder->files()->name('*.php')->in($directory) as $fileInfo) {
+			if (self::isFileLintSkipped($fileInfo->getPathname())) {
+				continue;
+			}
+
 			$files[$fileInfo->getBasename()] = [$fileInfo->getPathname()];
 		}
 

tests/bench/data/bug-10772.php

@@ -1,4 +1,4 @@
-<?php
+<?php // lint >= 8.1
 
 namespace Bug10772;
 

tests/bench/data/bug-10979.php

@@ -1,4 +1,4 @@
-<?php
+<?php // lint >= 8.1
 
 declare(strict_types=1);
 

tests/bench/data/bug-14869-enum.php

@@ -1,4 +1,4 @@
-<?php
+<?php // lint >= 8.1
 
 namespace Bug14869Enum;
 

tests/bench/data/conditional-expression-infinite-loop.php

@@ -1,4 +1,5 @@
-<?php declare(strict_types=1);
+<?php // lint >= 8.0
+declare(strict_types=1);
 
 namespace ConditionalExpressionInfiniteLoop;
 class test

tests/bench/data/hash-key-lookup.php

@@ -1,4 +1,5 @@
-<?php declare(strict_types = 1);
+<?php // lint >= 8.0
+declare(strict_types = 1);
 
 namespace BenchHashKeyLookup;
 

tests/bench/data/in-array-intersect-blowup.php

@@ -1,4 +1,5 @@
-<?php declare(strict_types = 1);
+<?php // lint >= 8.0
+declare(strict_types = 1);
 
 namespace BenchInArrayIntersectBlowup;