Skip to content

Commit afbc71d

Browse files
iluuu1994iluuu1994
committed
Add benchmarking to CI
Closes GH-11068
1 parent 5823955 commit afbc71d

File tree

6 files changed

+356
-0
lines changed

6 files changed

+356
-0
lines changed

‎.github/workflows/push.yml‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,3 +215,95 @@ jobs:
215215
run: .github/scripts/windows/build.bat
216216
- name: Test
217217
run: .github/scripts/windows/test.bat
218+
BENCHMARKING:
219+
name: BENCHMARKING
220+
runs-on: ubuntu-22.04
221+
steps:
222+
- name: git checkout
223+
uses: actions/checkout@v3
224+
with:
225+
fetch-depth: 0
226+
- name: apt
227+
run: |
228+
set -x
229+
sudo apt-get update
230+
sudo apt-get install \
231+
bison \
232+
libonig-dev \
233+
libsqlite3-dev \
234+
openssl \
235+
re2c \
236+
valgrind
237+
- name: ccache
238+
uses: hendrikmuhs/ccache-action@v1.2
239+
with:
240+
key: "${{github.job}}-${{hashFiles('main/php_version.h')}}"
241+
append-timestamp: false
242+
- name: ./configure
243+
run: |
244+
set -x
245+
./buildconf --force
246+
./configure \
247+
--disable-debug \
248+
--enable-mbstring \
249+
--enable-opcache \
250+
--enable-option-checking=fatal \
251+
--enable-sockets \
252+
--enable-werror \
253+
--prefix=/usr \
254+
--with-config-file-scan-dir=/etc/php.d \
255+
--with-mysqli=mysqlnd \
256+
--with-openssl \
257+
--with-pdo-sqlite \
258+
--with-valgrind
259+
- name: make
260+
run: make -j$(/usr/bin/nproc) >/dev/null
261+
- name: make install
262+
run: |
263+
set -x
264+
sudo make install
265+
sudo mkdir -p /etc/php.d
266+
sudo chmod 777 /etc/php.d
267+
echo mysqli.default_socket=/var/run/mysqld/mysqld.sock > /etc/php.d/mysqli.ini
268+
echo zend_extension=opcache.so >> /etc/php.d/opcache.ini
269+
echo opcache.enable=1 >> /etc/php.d/opcache.ini
270+
echo opcache.enable_cli=1 >> /etc/php.d/opcache.ini
271+
- name: Setup
272+
run: |
273+
git config --global user.name "Benchmark"
274+
git config --global user.email "benchmark@php.net"
275+
sudo service mysql start
276+
mysql -uroot -proot -e "CREATE DATABASE IF NOT EXISTS wordpress"
277+
mysql -uroot -proot -e "CREATE USER 'wordpress'@'localhost' IDENTIFIED BY 'wordpress'; FLUSH PRIVILEGES;"
278+
mysql -uroot -proot -e "GRANT ALL PRIVILEGES ON *.* TO 'wordpress'@'localhost' WITH GRANT OPTION;"
279+
- name: git checkout benchmarking-data
280+
uses: actions/checkout@v3
281+
with:
282+
repository: php/benchmarking-data
283+
ssh-key: ${{ secrets.BENCHMARKING_DATA_DEPLOY_KEY }}
284+
path: benchmark/repos/data
285+
- name: Benchmark
286+
run: php benchmark/benchmark.php true
287+
- name: Store result
288+
if: github.event_name == 'push'
289+
run: |
290+
set -x
291+
cd benchmark/repos/data
292+
git pull --autostash
293+
if [ -e ".git/MERGE_HEAD" ]; then
294+
echo "Merging, can't proceed"
295+
exit 1
296+
fi
297+
git add .
298+
if git diff --cached --quiet; then
299+
exit 0
300+
fi
301+
git commit -m "Add result for ${{ github.repository }}@${{ steps.determine-commit.outputs.sha }}"
302+
git push
303+
- name: Show diff
304+
if: github.event_name == 'pull_request'
305+
run: >-
306+
php benchmark/generate_diff.php
307+
${{ github.event.pull_request.head.sha }}
308+
$(git merge-base ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }})
309+
> $GITHUB_STEP_SUMMARY

‎benchmark/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
/repos

‎benchmark/benchmark.php‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
<?php
2+
3+
require_once __DIR__ . '/shared.php';
4+
5+
$storeResult = ($argv[1] ?? 'false') === 'true';
6+
$phpCgi = $argv[2] ?? dirname(PHP_BINARY) . '/php-cgi';
7+
if (!file_exists($phpCgi)) {
8+
fwrite(STDERR, "php-cgi not found\n");
9+
exit(1);
10+
}
11+
12+
function main() {
13+
global $storeResult;
14+
15+
$data = [];
16+
$data['Zend/bench.php'] = runBench(false);
17+
$data['Zend/bench.php JIT'] = runBench(true);
18+
$data['Symfony Demo 2.2.3'] = runSymfonyDemo(false);
19+
$data['Symfony Demo 2.2.3 JIT'] = runSymfonyDemo(true);
20+
$data['Wordpress 6.2'] = runWordpress(false);
21+
$data['Wordpress 6.2 JIT'] = runWordpress(true);
22+
$result = json_encode($data, JSON_PRETTY_PRINT) . "\n";
23+
24+
fwrite(STDOUT, $result);
25+
26+
if ($storeResult) {
27+
storeResult($result);
28+
}
29+
}
30+
31+
function storeResult(string $result) {
32+
$repo = __DIR__ . '/repos/data';
33+
cloneRepo($repo, 'git@github.com:php/benchmarking-data.git');
34+
35+
$commitHash = getPhpSrcCommitHash();
36+
$dir = $repo . '/' . substr($commitHash, 0, 2) . '/' . $commitHash;
37+
$summaryFile = $dir . '/summary.json';
38+
if (!is_dir($dir)) {
39+
mkdir($dir, 0755, true);
40+
}
41+
file_put_contents($summaryFile, $result);
42+
}
43+
44+
function getPhpSrcCommitHash(): string {
45+
$result = runCommand(['git', 'log', '--pretty=format:%H', '-n', '1'], dirname(__DIR__));
46+
return $result->stdout;
47+
}
48+
49+
function runBench(bool $jit): array {
50+
return runValgrindPhpCgiCommand([dirname(__DIR__) . '/Zend/bench.php'], jit: $jit);
51+
}
52+
53+
function runSymfonyDemo(bool $jit): array {
54+
$dir = __DIR__ . '/repos/symfony-demo-2.2.3';
55+
cloneRepo($dir, 'https://github.com/php/benchmarking-symfony-demo-2.2.3.git');
56+
runPhpCommand([$dir . '/bin/console', 'cache:clear']);
57+
runPhpCommand([$dir . '/bin/console', 'cache:warmup']);
58+
return runValgrindPhpCgiCommand([$dir . '/public/index.php'], cwd: $dir, jit: $jit, warmup: 50);
59+
}
60+
61+
function runWordpress(bool $jit): array {
62+
$dir = __DIR__ . '/repos/wordpress-6.2';
63+
cloneRepo($dir, 'https://github.com/php/benchmarking-wordpress-6.2.git');
64+
65+
/* FIXME: It might be better to use a stable version of PHP for this command because we can't
66+
* easily alter the phar file */
67+
runPhpCommand([
68+
'-d error_reporting=0',
69+
'wp-cli.phar',
70+
'core',
71+
'install',
72+
'--url=wordpress.local',
73+
'--title="Wordpress"',
74+
'--admin_user=wordpress',
75+
'--admin_password=wordpress',
76+
'--admin_email=benchmark@php.net',
77+
], $dir);
78+
79+
// Warmup
80+
runPhpCommand([$dir . '/index.php'], $dir);
81+
return runValgrindPhpCgiCommand([$dir . '/index.php'], cwd: $dir, jit: $jit, warmup: 50);
82+
}
83+
84+
function runPhpCommand(array $args, ?string $cwd = null): ProcessResult {
85+
return runCommand([PHP_BINARY, ...$args], $cwd);
86+
}
87+
88+
function runValgrindPhpCgiCommand(
89+
array $args,
90+
?string $cwd = null,
91+
bool $jit = false,
92+
int $warmup = 0,
93+
): array {
94+
global $phpCgi;
95+
$process = runCommand([
96+
'valgrind',
97+
'--tool=callgrind',
98+
'--dump-instr=yes',
99+
'--callgrind-out-file=/dev/null',
100+
'--',
101+
$phpCgi,
102+
'-T' . ($warmup ? $warmup . ',' : '') . '1',
103+
'-d max_execution_time=0',
104+
'-d opcache.enable=1',
105+
'-d opcache.jit_buffer_size=' . ($jit ? '128M' : '0'),
106+
...$args,
107+
]);
108+
$instructions = extractInstructionsFromValgrindOutput($process->stderr);
109+
return ['instructions' => $instructions];
110+
}
111+
112+
function extractInstructionsFromValgrindOutput(string $output): string {
113+
preg_match("(==[0-9]+== Events : Ir\n==[0-9]+== Collected : (?<instructions>[0-9]+))", $output, $matches);
114+
return $matches['instructions'] ?? throw new \Exception('Unexpected valgrind output');
115+
}
116+
117+
main();

‎benchmark/docker-compose.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
version: "3.8"
2+
services:
3+
wordpress_db:
4+
image: mysql:8.0
5+
ports:
6+
- "3306:3306"
7+
environment:
8+
MYSQL_ROOT_PASSWORD: root
9+
MYSQL_DATABASE: wordpress
10+
MYSQL_USER: wordpress
11+
MYSQL_PASSWORD: wordpress

‎benchmark/generate_diff.php‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
<?php
2+
3+
require_once __DIR__ . '/shared.php';
4+
5+
function main(?string $headCommitHash, ?string $baseCommitHash) {
6+
if ($headCommitHash === null || $baseCommitHash === null) {
7+
fwrite(STDERR, "Usage: php generate_diff.php HEAD_COMMIT_HASH BASE_COMMIT_HASH\n");
8+
exit(1);
9+
}
10+
11+
$repo = __DIR__ . '/repos/data';
12+
cloneRepo($repo, 'git@github.com:php/benchmarking-data.git');
13+
$headSummaryFile = $repo . '/' . substr($headCommitHash, 0, 2) . '/' . $headCommitHash . '/summary.json';
14+
$baseSummaryFile = $repo . '/' . substr($baseCommitHash, 0, 2) . '/' . $baseCommitHash . '/summary.json';
15+
if (!file_exists($headSummaryFile)) {
16+
return "Head commit '$headCommitHash' not found\n";
17+
}
18+
if (!file_exists($baseSummaryFile)) {
19+
return "Base commit '$baseCommitHash' not found\n";
20+
}
21+
$headSummary = json_decode(file_get_contents($headSummaryFile), true);
22+
$baseSummary = json_decode(file_get_contents($baseSummaryFile), true);
23+
24+
$headCommitHashShort = substr($headCommitHash, 0, 7);
25+
$baseCommitHashShort = substr($baseCommitHash, 0, 7);
26+
$output = "| Benchmark | Base ($baseCommitHashShort) | Head ($headCommitHashShort) | Diff |\n";
27+
$output .= "|---|---|---|---|\n";
28+
foreach ($headSummary as $name => $headBenchmark) {
29+
$baseInstructions = $baseSummary[$name]['instructions'] ?? null;
30+
$headInstructions = $headSummary[$name]['instructions'];
31+
$output .= "| $name | "
32+
. formatInstructions($baseInstructions) . " | "
33+
. formatInstructions($headInstructions) . " | "
34+
. formatDiff($baseInstructions, $headInstructions) . " |\n";
35+
}
36+
return $output;
37+
}
38+
39+
function formatInstructions(?int $instructions): string {
40+
if ($instructions === null) {
41+
return '-';
42+
}
43+
if ($instructions > 1e6) {
44+
return sprintf('%.0fM', $instructions / 1e6);
45+
} elseif ($instructions > 1e3) {
46+
return sprintf('%.0fK', $instructions / 1e3);
47+
} else {
48+
return (string) $instructions;
49+
}
50+
}
51+
52+
function formatDiff(?int $baseInstructions, int $headInstructions): string {
53+
if ($baseInstructions === null) {
54+
return '-';
55+
}
56+
$instructionDiff = $headInstructions - $baseInstructions;
57+
return sprintf('%.2f%%', $instructionDiff / $baseInstructions * 100);
58+
}
59+
60+
$headCommitHash = $argv[1] ?? null;
61+
$baseCommitHash = $argv[2] ?? null;
62+
$output = main($headCommitHash, $baseCommitHash);
63+
fwrite(STDOUT, $output);

‎benchmark/shared.php‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
<?php
2+
3+
class ProcessResult {
4+
public $stdout;
5+
public $stderr;
6+
}
7+
8+
function runCommand(array $args, ?string $cwd = null): ProcessResult {
9+
$cmd = implode(' ', array_map('escapeshellarg', $args));
10+
$pipes = null;
11+
$result = new ProcessResult();
12+
$descriptorSpec = [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']];
13+
fwrite(STDOUT, "> $cmd\n");
14+
$processHandle = proc_open($cmd, $descriptorSpec, $pipes, $cwd ?? getcwd(), null);
15+
16+
$stdin = $pipes[0];
17+
$stdout = $pipes[1];
18+
$stderr = $pipes[2];
19+
20+
fclose($stdin);
21+
22+
stream_set_blocking($stdout, false);
23+
stream_set_blocking($stderr, false);
24+
25+
$stdoutEof = false;
26+
$stderrEof = false;
27+
28+
do {
29+
$read = [$stdout, $stderr];
30+
$write = null;
31+
$except = null;
32+
33+
stream_select($read, $write, $except, 1, 0);
34+
35+
foreach ($read as $stream) {
36+
$chunk = fgets($stream);
37+
if ($stream === $stdout) {
38+
$result->stdout .= $chunk;
39+
} elseif ($stream === $stderr) {
40+
$result->stderr .= $chunk;
41+
}
42+
}
43+
44+
$stdoutEof = $stdoutEof || feof($stdout);
45+
$stderrEof = $stderrEof || feof($stderr);
46+
} while(!$stdoutEof || !$stderrEof);
47+
48+
fclose($stdout);
49+
fclose($stderr);
50+
51+
$statusCode = proc_close($processHandle);
52+
if ($statusCode !== 0) {
53+
fwrite(STDOUT, $result->stdout);
54+
fwrite(STDERR, $result->stderr);
55+
fwrite(STDERR, 'Exited with status code ' . $statusCode . "\n");
56+
exit($statusCode);
57+
}
58+
59+
return $result;
60+
}
61+
62+
function cloneRepo(string $path, string $url) {
63+
if (is_dir($path)) {
64+
return;
65+
}
66+
$dir = dirname($path);
67+
$repo = basename($path);
68+
if (!is_dir($dir)) {
69+
mkdir($dir, 0755, true);
70+
}
71+
runCommand(['git', 'clone', '-q', '--end-of-options', $url, $repo], dirname($path));
72+
}

0 commit comments

Comments
 (0)